Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CI: split Repo tests (CPU, rest) and (CPU, gloo) before they reach their 2100 s cap (#15951)

Daniel Han4a87de122efe
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 3d528e68fca..e2b51fe8b92 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -350,9 +350,24 @@ jobs:
--ignore=tests/sh
--ignore=tests/vllm_compat
--ignore=tests/version_compat
+ --ignore=tests/diffusion
+ --ignore=tests/kaggle
+ --ignore=tests/security
+ --ignore-glob=tests/test_[dimsu]*.py
--ignore-glob=tests/test_fsdp2*.py
--ignore-glob=tests/test_[!f]*gloo*.py
--ignore-glob=tests/test_f[!s]*gloo*.py
+ # The catch-all then ran 1213 to 2092 s of its 2100 s cap on 2026-10-29 (median about
+ # 1750 s), and was interrupted after passing on a slow runner. Measured per file, the
+ # diffusion, kaggle and security folders plus top-level test_d/i/m/s/u files are about
+ # half of it. No *gloo* or fsdp2 file starts with d, i, m, s or u, so the glob cannot
+ # take a file another shard also names (tests/test_ci_repo_cpu_shards.py checks it).
+ - shard: rest-2
+ selection: >-
+ tests/diffusion
+ tests/kaggle
+ tests/security
+ tests/test_[dimsu]*.py
--deselect tests/test_model_registry.py::test_model_registration
--deselect tests/test_model_registry.py::test_all_model_registration
# Native FSDP2's gloo tests spawn 2 to 4 ranks each: about ten minutes of this
@@ -379,8 +394,18 @@ jobs:
# the globs themselves.
- shard: gloo
selection: >-
- tests/test_[!f]*gloo*.py
+ tests/test_[!fpt]*gloo*.py
tests/test_f[!s]*gloo*.py
+ tests/test_p[!p]*gloo*.py
+ tests/test_tp_[!n]*gloo*.py
+ # The gloo shard then ran 1331 to 2058 s of the same cap on 2026-10-29. Measured per
+ # file, native tensor parallel plus pipeline parallel are about 51% of it, everything
+ # else about 49%. A *gloo* file named test_t[!p]* or test_tp[!_]* would match neither
+ # shard; tests/test_ci_repo_cpu_shards.py reports it as dropped.
+ - shard: gloo-tp-pp
+ selection: >-
+ tests/test_tp_n*gloo*.py
+ tests/test_pp*gloo*.py
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.py
index c0af1c0a9e6..164ea30c5a6 100644
--- a/tests/studio/test_backend_ci_parallel_isolation.py
+++ b/tests/studio/test_backend_ci_parallel_isolation.py
@@ -81,11 +81,17 @@ def _collects(command: str, path: str) -> bool:
def under(prefix, candidate):
return candidate == prefix or candidate.startswith(prefix + "/")
+ def rooted(root, candidate):
+ # A glob root (tests/test_fsdp2_*.py) is expanded by bash before pytest sees it.
+ if any(char in root for char in "*?["):
+ return fnmatch.fnmatch(candidate, root)
+ return under(root, candidate)
+
if any(under(ignore, path) for ignore in ignores):
return False
if any(_ignore_glob_hits(pattern, path) for pattern in globs):
return False
- return any(under(root, path) for root in roots)
+ return any(rooted(root, path) for root in roots)
def _ignore_glob_hits(pattern: str, path: str) -> bool:
@@ -318,15 +324,15 @@ def test_the_command_scan_sees_the_parallel_run_and_the_serial_steps():
"""Pin the parser: a scan that matched nothing would pass both tests above."""
commands = _pytest_commands()
parallel = [command for command in commands if " -n " in f" {command} "]
- assert len(parallel) == 9, (
- f"expected nine parallel pytest runs, three shards of the backend matrix and six "
+ assert len(parallel) == 11, (
+ f"expected eleven parallel pytest runs, three shards of the backend matrix and eight "
f"of repo-cpu-tests, got {parallel}. If a job stopped running in parallel, or a "
f"shard was added or removed, say so here rather than letting this scan quietly "
f"cover fewer runs."
)
root = [command for command in parallel if _over_the_repo_tests(command)]
- assert len(root) == 6, (
- f"expected the six repo-root shards, got {root}. The isolation checks above apply "
+ assert len(root) == 8, (
+ f"expected the eight repo-root shards, got {root}. The isolation checks above apply "
f"to those, and a scan that matched none of them would pass on nothing."
)
backend = [command for command in parallel if _over_the_backend(command)]
diff --git a/tests/test_ci_repo_cpu_shards.py b/tests/test_ci_repo_cpu_shards.py
index 669bf8587ca..826613261aa 100644
--- a/tests/test_ci_repo_cpu_shards.py
+++ b/tests/test_ci_repo_cpu_shards.py
@@ -148,16 +148,19 @@ class TestEveryGlobRootMatchesAFile:
class TestTheSplitKeepsTheSelectionItInherited:
def test_the_deselected_hub_tests_stay_deselected(self):
- """Both hit huggingface_hub for live model existence checks. They live in a shard-3
- file, so shard 3 is where the deselect has to be."""
- selection = _selection_text()
+ """Both hit huggingface_hub for live model existence checks. The deselect only acts in
+ the shard that collects their file, so it is checked there, not anywhere in the matrix."""
+ owners = _claiming_shards("tests/test_model_registry.py", _shards())
+ assert len(owners) == 1, f"tests/test_model_registry.py is collected by {owners}"
+ selection = _selection_text(owners[0])
for test_id in (
"tests/test_model_registry.py::test_model_registration",
"tests/test_model_registry.py::test_all_model_registration",
):
- assert (
- f"--deselect {test_id}" in selection
- ), f"{test_id} reaches the network and must stay deselected"
+ assert f"--deselect {test_id}" in selection, (
+ f"{test_id} reaches the network and must stay deselected in {owners[0]}, the "
+ "shard that runs its file"
+ )
def test_every_shard_keeps_the_marker_filter(self):
"""`server` needs a studio venv and `e2e` needs the network. The filter is on the
@@ -183,10 +186,11 @@ class TestTheGuardIsNotVacuous:
assert _claiming_shards("tests/studio/test_two_owners.py", broken) == ["one", "two"]
-def _selection_text() -> str:
+def _selection_text(shard: str | None = None) -> str:
return " ".join(
" ".join(entry["selection"].split())
for entry in _backend_ci()["jobs"]["repo-cpu-tests"]["strategy"]["matrix"]["include"]
+ if shard is None or entry["shard"] == shard
)
diff --git a/tests/test_cli_export_unpacking.py b/tests/test_cli_export_unpacking.py
index 8d201a38297..46dad7b83b0 100644
--- a/tests/test_cli_export_unpacking.py
+++ b/tests/test_cli_export_unpacking.py
@@ -51,6 +51,9 @@ class _FakeExportBackend:
def _install_fake_studio_backend(monkeypatch: pytest.MonkeyPatch) -> None:
"""Inject a fake studio.backend.core.export into sys.modules so the CLI's lazy import binds to it; parent packages stubbed to skip the structlog-dependent tree."""
+ # The CLI puts studio/backend on sys.path, whose tests/ package then shadows the repo's for any
+ # process spawned later in this worker (`No module named 'tests.<file>'`): restore it afterwards.
+ monkeypatch.setattr(sys, "path", list(sys.path))
# Load the real CLI first: the stubs below shadow studio.backend.utils, which it imports.
importlib.import_module("unsloth_cli.commands.export")
@@ -88,6 +91,34 @@ def runner() -> CliRunner:
return CliRunner()
+def test_an_export_run_leaves_sys_path_as_it_found_it(runner: CliRunner, tmp_path: Path) -> None:
+ """The run puts studio/backend first on sys.path, and studio/backend/tests is a package: left
+ there, every later `spawn` child in the worker resolves `tests.<file>` to it and dies. First in
+ the file, so the tests below cannot have leaked the path before it looks."""
+ before = list(sys.path)
+ ckpt = tmp_path / "ckpt"
+ ckpt.mkdir()
+ backend_dir = str(Path(__file__).resolve().parents[1] / "studio" / "backend")
+ with pytest.MonkeyPatch.context() as monkeypatch:
+ _install_fake_studio_backend(monkeypatch)
+ # Another collected module may already have put it there, and the CLI skips a path it finds.
+ # In place, on the copy the helper above swapped in: only that helper's restore may undo it.
+ sys.path[:] = [p for p in sys.path if p != backend_dir]
+ clean = list(sys.path)
+ from unsloth_cli.commands import export as export_cmd
+
+ app = typer.Typer()
+ app.command("export")(export_cmd.export)
+ app.command("noop")(lambda: None)
+ result = runner.invoke(
+ app, ["export", str(ckpt), str(tmp_path / "out"), "--format", "lora"]
+ )
+ assert result.exit_code == 0, result.output
+ # Non-vacuous: the run did put studio/backend on sys.path, which the restore has to undo.
+ assert backend_dir in sys.path and backend_dir not in clean
+ assert sys.path == before
+
+
@pytest.mark.parametrize(
"format_flag,quant_flag",
[