Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Split Repo tests (CPU) into three shards, and take the shell suite off its critical path (#10882)
Daniel Han7f6219e3af49
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.ymlindex 3f60ab9a965..0762b724a2e 100644--- a/.github/workflows/studio-backend-ci.yml+++ b/.github/workflows/studio-backend-ci.yml@@ -352,7 +352,7 @@ jobs:# pre-loads unsloth_zoo.device_type and unsloth.device_type under a# mocked torch.cuda.is_available so the unsloth import chain# succeeds on CPU.- name: Repo tests (CPU)+ name: Repo tests (CPU, ${{ matrix.shard }})runs-on: ubuntu-latest# Same reason as the matrix above, and now the same number. The comment this replaces# claimed 13.4 minutes. Across the last 40 runs of this job on main the pytest step@@ -369,7 +369,60 @@ jobs:# the run blames whichever step was unlucky -- on both of those it was "Shell installer# tests", 84s in and unrelated -- and the shell, CLI and Docker steps never run at all,# so the PR loses that coverage silently. The suite is not the problem; the budget was.+ #+ # The cap stays at 45 through the split below rather than being retuned on a prediction:+ # a cap that lands inside the real spread is exactly the failure this number was raised+ # to fix, and the sharded spread has not been measured on a runner yet.timeout-minutes: 45+ strategy:+ # Three shards of one selection, balanced on measured time, not on file count. Summed+ # test time over the whole discovered set is 2946s (`--durations=0`, -n 16), and it+ # splits by directory almost exactly in thirds:+ #+ # tests/studio 1035.8s 271 files 35.2%+ # tests/python + tests/kaggle 955.8s 124 files 32.4%+ # everything else 954.4s 180 files 32.4%+ #+ # so the longest shard is 1.056x an ideal third and no rebalancing by named file is+ # worth the property below.+ #+ # Shard 3 is written as "tests/ minus the other two roots", never as an allowlist, so+ # a new file lands in exactly one shard without a workflow edit: under tests/studio it+ # is shard 1's, under tests/python or tests/kaggle shard 2's, and anywhere else,+ # including a brand-new top-level directory, shard 3's.+ # tests/test_ci_repo_cpu_shards.py fails if that stops being true.+ #+ # fail-fast off: three shards of one suite, and cancelling the other two on the first+ # failure would hide which of them were also red.+ fail-fast: false+ matrix:+ include:+ - shard: studio+ selection: >-+ tests/studio+ --ignore=tests/studio/load_freeze+ --ignore=tests/studio/test_hardware_dispatch_matrix.py+ --ignore=tests/studio/test_is_mlx_dispatch_gate.py+ --ignore=tests/studio/test_xpu_spoof_pipeline.py+ --ignore=tests/studio/test_mlx_context_platform_matrix.py+ - shard: python+ selection: >-+ tests/python+ tests/kaggle+ - shard: rest+ selection: >-+ tests/+ --ignore=tests/studio+ --ignore=tests/python+ --ignore=tests/kaggle+ --ignore=tests/qlora+ --ignore=tests/saving+ --ignore=tests/utils+ --ignore=tests/sh+ --ignore=tests/vllm_compat+ --ignore=tests/version_compat+ --deselect tests/test_model_registry.py::test_model_registration+ --deselect tests/test_model_registry.py::test_all_model_registrationsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:@@ -504,23 +557,12 @@ jobs:# under the job cap, so this fires first and says where it was stuck.run: |timeout --signal=INT --kill-after=60 2100 \- python -m pytest tests/ -q --tb=short -n 4 --timeout=330 \- --ignore=tests/qlora \- --ignore=tests/saving \- --ignore=tests/utils \- --ignore=tests/sh \- --ignore=tests/studio/load_freeze \- --ignore=tests/studio/test_hardware_dispatch_matrix.py \- --ignore=tests/studio/test_is_mlx_dispatch_gate.py \- --ignore=tests/studio/test_xpu_spoof_pipeline.py \- --ignore=tests/studio/test_mlx_context_platform_matrix.py \- --ignore=tests/vllm_compat \- --ignore=tests/version_compat \- -m 'not server and not e2e' \- --deselect tests/test_model_registry.py::test_model_registration \- --deselect tests/test_model_registry.py::test_all_model_registration+ python -m pytest ${{ matrix.selection }} -q --tb=short -n 4 --timeout=330 \+ -m 'not server and not e2e'- name: Hardware-spoof tests (state-sensitive, run in isolation)+ # The four spoof files live under tests/studio, so they belong to that shard.+ if: matrix.shard == 'studio'env:PYTHONPATH: ${{ github.workspace }}/studioUNSLOTH_COMPILE_DISABLE: '1'@@ -536,6 +578,9 @@ jobs:tests/studio/test_mlx_context_platform_matrix.py- name: Event-loop latency tests (wall clock, run without CPU contention)+ # tests/studio/load_freeze, so the studio shard, and it still runs alone in its own+ # invocation: two shards on two runners do not contend with each other.+ if: matrix.shard == 'studio'env:PYTHONPATH: ${{ github.workspace }}/studioUNSLOTH_COMPILE_DISABLE: '1'@@ -555,6 +600,9 @@ jobs:run: python -m pytest tests/studio/load_freeze -q --tb=short- name: CLI tests (unsloth_cli)+ # Not under tests/, so no shard owns it by discovery; it rides the python shard,+ # which is the shortest of the three.+ if: matrix.shard == 'python'# unsloth_cli/tests had no CI at all: `unsloth_cli/**` was only a paths# trigger and a ruff target, so 673 tests covering the studio launcher,# the pre-exposure gate and the auth secret writers ran nowhere, and@@ -564,6 +612,44 @@ jobs:# (it self-bootstraps sys.path and imports neither unsloth nor torch).run: python -m pytest unsloth_cli/tests -q --tb=short+ - name: Docker JupyterLab/notebook feature validation+ # Named validate_studio_features.py (not test_*.py) so pytest skips it;+ # run explicitly so notebook/Colab/branding regressions fail CI.+ if: matrix.shard == 'rest'+ run: python tests/validate_studio_features.py++ - name: Save the pip cache+ if: always()+ uses: ./.github/actions/pip-cache-save+ with:+ dir: ${{ steps.pip-cache.outputs.dir }}+ key: ${{ steps.pip-cache.outputs.key }}+ cache-hit: ${{ steps.pip-cache.outputs.cache-hit }}++ shell-installer-tests:+ # Its own job rather than the tail of the longest one. 64 hermetic bash suites, run+ # serially at 158-165s measured, sharing nothing with the Python install they used to sit+ # behind: they mock uv, wsl.exe and the venv, and extract functions out of install.sh with+ # sed. Two consequences of the old placement, both measured: those 165s were on the+ # critical path of a job that was already the longest on the board, and when that job hit+ # its cap mid-run it was this step that got named (#10849, #10861) while the steps after+ # it silently never ran.+ #+ # No pip install here at all -- the runner's own python3 and bash are what these scripts+ # use -- so the job is checkout plus the loop, and it finishes long before the shards do.+ name: Shell installer tests+ runs-on: ubuntu-latest+ # 165s measured, and a hung mock would otherwise sit here until the workflow's own limit.+ timeout-minutes: 15+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: '3.12'+- name: Shell installer tests# Auto-discovered rather than allowlisted. The old hardcoded list had# silently fallen seven files behind tests/run_all.sh, including@@ -590,16 +676,3 @@ jobs:done[ "$found" -gt 0 ] || { echo "::error::no shell tests discovered under tests/sh"; exit 1; }echo "ran $found shell installer test files"-- - name: Docker JupyterLab/notebook feature validation- # Named validate_studio_features.py (not test_*.py) so pytest skips it;- # run explicitly so notebook/Colab/branding regressions fail CI.- run: python tests/validate_studio_features.py-- - name: Save the pip cache- if: always()- uses: ./.github/actions/pip-cache-save- with:- dir: ${{ steps.pip-cache.outputs.dir }}- key: ${{ steps.pip-cache.outputs.key }}- cache-hit: ${{ steps.pip-cache.outputs.cache-hit }}diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.pyindex c1a7239745c..3199f6a6c0c 100644--- a/tests/studio/test_backend_ci_parallel_isolation.py+++ b/tests/studio/test_backend_ci_parallel_isolation.py@@ -27,6 +27,7 @@ import refrom pathlib import Pathimport pytest+import yamlWORKFLOW = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "studio-backend-ci.yml"@@ -46,25 +47,70 @@ def _pytest_commands(text: str) -> list[str]:Read off the raw text rather than the parsed YAML: a `run:` block is onescalar and the interesting structure is inside it, so parsing buys nothing andwould make this depend on the job/step layout instead of the commands.++ The one thing that cannot be read off the text is `${{ matrix.selection }}`: since+ repo-cpu-tests became three shards, its parallel command is a template and its paths+ live in the matrix. Those are substituted in, one command per shard, so a scan of the+ commands is still a scan of what actually runs."""joined = re.sub(r"\\\s*\n\s*", " ", text)- return [+ commands = [line.strip()for line in joined.splitlines()if "python -m pytest" in line and not line.lstrip().startswith("#")]+ expanded = []+ for command in commands:+ if "${{ matrix.selection }}" not in command:+ expanded.append(command)+ continue+ for selection in _shard_selections():+ expanded.append(command.replace("${{ matrix.selection }}", selection))+ return expanded+++def _shard_selections() -> list[str]:+ job = yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))["jobs"]["repo-cpu-tests"]+ return [" ".join(entry["selection"].split()) for entry in job["strategy"]["matrix"]["include"]]+++def _collects(command: str, path: str) -> bool:+ """Whether a pytest command would collect `path`, by its roots and its --ignore flags.++ An isolated path used to be kept out of the parallel run by naming it in an --ignore.+ A shard that does not name its directory at all keeps it out just as effectively, so+ the question the guard asks is whether the run reaches the path, not how."""+ tokens = command.split()+ roots, ignores = [], []+ index = 0+ while index < len(tokens):+ token = tokens[index]+ if token.startswith("--ignore="):+ ignores.append(token.split("=", 1)[1].rstrip("/"))+ elif token == "--deselect":+ index += 1+ elif token.startswith("tests/") or token == "tests":+ roots.append(token.rstrip("/"))+ index += 1++ def under(prefix, candidate):+ return candidate == prefix or candidate.startswith(prefix + "/")++ if any(under(ignore, path) for ignore in ignores):+ return False+ return any(under(root, path) for root in roots)# Two different trees are run in parallel now, and the isolation below belongs to exactly one of them. The repo-root# job runs `tests/` from the checkout; the matrix job runs the backend's own suite with# `working-directory: studio/backend`, so `tests/studio/...` is not a path that exists for it.-# Told apart by what they ignore, because that is the thing both the ignores and this scan are about: only the-# repo-root run excludes directories that live at the repo root.-REPO_ROOT_MARKER = "--ignore=tests/qlora"+# Told apart by the selection they carry. This used to be one flag, `--ignore=tests/qlora`, which only the repo-root+# run had; the repo-root run is three shards now and only one of them carries that flag, so the test is membership of+# the matrix instead. The backend matrix run is not in it.-def _over_the_repo_root(command: str) -> bool:- return REPO_ROOT_MARKER in command+def _over_the_repo_tests(command: str) -> bool:+ return any(selection in command for selection in _shard_selections())# The same pairing, for the backend matrix run. Ignoring a file from the parallel run and running it again serially is@@ -317,10 +363,10 @@ def _fragile_timing_asserts(path: Path) -> list:@pytest.mark.parametrize("path, reason", ISOLATED, ids = [p for p, _ in ISOLATED])def test_an_isolated_path_is_ignored_by_every_parallel_pytest_run(path, reason):for command in _pytest_commands(WORKFLOW.read_text(encoding = "utf-8")):- if " -n " not in f" {command} " or not _over_the_repo_root(command):+ if " -n " not in f" {command} " or not _over_the_repo_tests(command):continue- assert f"--ignore={path}" in command, (- f"{path} ({reason}) is not ignored by a parallel pytest run in "+ assert not _collects(command, path), (+ f"{path} ({reason}) is collected by a parallel pytest run in "f"{WORKFLOW.name}, so it shares four workers on the runner's four vCPUs: {command}")@@ -344,20 +390,24 @@ def test_the_command_scan_sees_the_parallel_run_and_the_serial_steps():"""Pin the parser: a scan that matched nothing would pass both tests above."""commands = _pytest_commands(WORKFLOW.read_text(encoding = "utf-8"))parallel = [command for command in commands if " -n " in f" {command} "]- assert len(parallel) == 2, (- f"expected two parallel pytest runs, the backend matrix and repo-cpu-tests, got "- f"{parallel}. If a job stopped running in parallel, say so here rather than "- f"letting this scan quietly cover one run."+ assert len(parallel) == 4, (+ f"expected four parallel pytest runs, the backend matrix and repo-cpu-tests' three "+ f"shards, got {parallel}. If a job stopped running in parallel, or a shard was added "+ f"or removed, say so here rather than letting this scan quietly cover fewer runs.")- root = [command for command in parallel if _over_the_repo_root(command)]- assert len(root) == 1, (- f"expected exactly one parallel run over the repo root, got {root}. The isolation "- f"checks above apply to that one, and a scan that matched none of them would pass "- f"on nothing."+ root = [command for command in parallel if _over_the_repo_tests(command)]+ assert len(root) == 3, (+ f"expected the three repo-root shards, got {root}. The isolation checks above apply "+ f"to those, and a scan that matched none of them would pass on nothing.")- # The line joins have to be resolved, or the parallel command reads as `pytest tests/ -q` with none of its --ignore- # flags and the first test above passes on nothing.- assert "--ignore=" in root[0]+ # The line joins and the matrix substitution both have to be resolved, or a shard command reads as+ # `pytest ${{ matrix.selection }} -q` with no paths at all and the first test above passes on nothing.+ assert all("${{" not in command for command in root)+ assert any("--ignore=" in command for command in root)+ assert {command for command in root} == set(root), "a shard selection appears twice"+ # Non-vacuous the other way too: the shards between them must reach the repo's test root, or "not collected by any+ # parallel run" would be true of every path in the repo.+ assert any(_collects(command, "tests/test_model_registry.py") for command in root)assert len(commands) > 1, "no serial pytest steps found; the ignore checks cannot fail"diff --git a/tests/test_ci_repo_cpu_shards.py b/tests/test_ci_repo_cpu_shards.pynew file mode 100644index 00000000000..f81ab4f31b0--- /dev/null+++ b/tests/test_ci_repo_cpu_shards.py@@ -0,0 +1,198 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Guards that the sharded `Repo tests (CPU)` job still runs every test it used to.++`Repo tests (CPU)` was one job discovering all of `tests/`. It is now three shards on three+runners, and the whole risk of that is a test file landing in no shard: the job goes green+faster having checked less, and nothing says so.++So the split is not three allowlists. Two shards name their roots and the third is+"everything under tests/ except those roots", which makes exactly-one-shard a property of+the shape rather than of anyone remembering to edit the workflow. These tests hold that+shape: every directory under tests/ is claimed once, the catch-all really is a catch-all,+and the directories no shard runs are the ones that were already excluded, each still+excluded for the reason another job covers it.++The per-test-id partition was checked directly when the split landed, by running each shard+and the old selection and diffing their JUnit reports: 8263 + 3559 + 7234 = 19056 ids, no+overlap, no gap, and every id in the same state in both. That is a measurement of one tree;+these tests are what keeps it true of the next one.+"""++from pathlib import Path++import pytest+import yaml+++REPO_ROOT = Path(__file__).resolve().parents[1]++_BACKEND_CI = REPO_ROOT / ".github" / "workflows" / "studio-backend-ci.yml"+_TESTS_DIR = REPO_ROOT / "tests"++# Directories under tests/ that no shard runs, and what runs them instead. A shard losing a+# directory would otherwise look exactly like one of these.+_NOT_IN_ANY_SHARD = {+ "tests/qlora": "GPU-bound: needs real weights",+ "tests/saving": "GPU-bound: needs real weights",+ "tests/utils": "a helpers folder, not a suite",+ "tests/sh": "the shell suite, run by the shell-installer-tests job",+ "tests/vllm_compat": "multi-version drift canary, run by version-compat-ci.yml",+ "tests/version_compat": "multi-version drift canary, run by version-compat-ci.yml",+}+++def _backend_ci() -> dict:+ return yaml.safe_load(_BACKEND_CI.read_text(encoding = "utf-8"))+++def _shards() -> dict:+ """{shard name: (roots, ignores)} parsed out of the job's matrix."""+ job = _backend_ci()["jobs"]["repo-cpu-tests"]+ shards = {}+ for entry in job["strategy"]["matrix"]["include"]:+ roots, ignores = [], []+ tokens = entry["selection"].split()+ index = 0+ while index < len(tokens):+ token = tokens[index]+ if token.startswith("--ignore="):+ ignores.append(token.split("=", 1)[1].rstrip("/"))+ elif token == "--deselect":+ index += 1 # a test id, not a path+ elif token.startswith("-"):+ pass+ else:+ roots.append(token.rstrip("/"))+ index += 1+ shards[entry["shard"]] = (roots, ignores)+ return shards+++def _covers(prefix: str, path: str) -> bool:+ prefix = prefix.rstrip("/")+ return path == prefix or path.startswith(prefix + "/")+++def _claiming_shards(path: str, shards: dict) -> list:+ """Which shards would collect `path`, by pytest's own root-and-ignore rules."""+ claiming = []+ for name, (roots, ignores) in shards.items():+ if any(_covers(ignore, path) for ignore in ignores):+ continue+ if any(_covers(root, path) for root in roots):+ claiming.append(name)+ return sorted(claiming)+++def _isolated_paths() -> list:+ """Paths the job runs in their own pytest invocation instead of inside a shard.++ The spoof files mutate hardware.py module globals and tests/studio/load_freeze asserts+ wall-clock bounds, so both are ignored by their shard's discovery and named by a step of+ their own. Read out of those steps rather than listed here, so moving one between the+ two places cannot make it look dropped.+ """+ paths = []+ for step in _backend_ci()["jobs"]["repo-cpu-tests"]["steps"]:+ run = str(step.get("run", ""))+ if "${{ matrix.selection }}" in run or "pytest" not in run:+ continue+ paths.extend(token for token in run.split() if token.startswith("tests/"))+ return paths+++def _test_files() -> list:+ return sorted(str(path.relative_to(REPO_ROOT)) for path in _TESTS_DIR.rglob("test_*.py"))+++class TestEveryTestFileLandsInExactlyOneShard:+ def test_no_file_is_dropped_or_run_twice(self):+ shards = _shards()+ isolated = _isolated_paths()+ dropped, doubled = [], []+ for path in _test_files():+ if any(_covers(excluded, path) for excluded in _NOT_IN_ANY_SHARD):+ continue+ if any(_covers(named, path) for named in isolated):+ continue # run by its own step, for a state or timing reason+ claiming = _claiming_shards(path, shards)+ if not claiming:+ dropped.append(path)+ elif len(claiming) > 1:+ doubled.append((path, claiming))+ assert not dropped, (+ "these files are collected by no shard, so Repo tests (CPU) no longer runs "+ f"them: {dropped}"+ )+ assert not doubled, f"these files are collected by more than one shard: {doubled}"++ def test_a_brand_new_top_level_directory_is_claimed(self):+ """The property the catch-all exists for: a directory nobody has heard of yet still+ runs, without a workflow edit."""+ claiming = _claiming_shards("tests/a_directory_added_tomorrow/test_new.py", _shards())+ assert claiming == ["rest"], (+ "a new directory under tests/ must land in the catch-all shard, and only there; "+ f"got {claiming}"+ )++ @pytest.mark.parametrize("excluded", sorted(_NOT_IN_ANY_SHARD))+ def test_the_directories_no_shard_runs_are_the_expected_ones(self, excluded):+ """Excluded on purpose, and still excluded from every shard rather than from one."""+ claiming = _claiming_shards(f"{excluded}/test_anything.py", _shards())+ assert claiming == [], (+ f"{excluded} is {_NOT_IN_ANY_SHARD[excluded]}, but shard(s) {claiming} now "+ "collect it"+ )+++class TestTheSplitKeepsTheSelectionItInherited:+ def test_the_deselected_hub_tests_stay_deselected(self):+ """Both hit huggingface_hub for live model existence checks. They live in a shard-3+ file, so shard 3 is where the deselect has to be."""+ selection = _selection_text()+ for test_id in (+ "tests/test_model_registry.py::test_model_registration",+ "tests/test_model_registry.py::test_all_model_registration",+ ):+ assert (+ f"--deselect {test_id}" in selection+ ), f"{test_id} reaches the network and must stay deselected"++ def test_every_shard_keeps_the_marker_filter(self):+ """`server` needs a studio venv and `e2e` needs the network. The filter is on the+ shared step rather than per shard, so it is checked there."""+ step = _shard_pytest_step()+ assert "-m 'not server and not e2e'" in step["run"]++ def test_every_shard_keeps_the_timeouts(self):+ """A shard that loses these reports "cancelled" and names nothing, which is what+ #9515 / #9530 were about."""+ run = _shard_pytest_step()["run"]+ assert "--timeout=330" in run+ assert "timeout --signal=INT --kill-after=60" in run+++class TestTheGuardIsNotVacuous:+ def test_a_gap_is_reported(self):+ broken = {"one": (["tests/studio"], []), "two": (["tests/python"], [])}+ assert _claiming_shards("tests/test_orphan.py", broken) == []++ def test_an_overlap_is_reported(self):+ broken = {"one": (["tests/"], []), "two": (["tests/studio"], [])}+ assert _claiming_shards("tests/studio/test_two_owners.py", broken) == ["one", "two"]+++def _selection_text() -> str:+ return " ".join(+ " ".join(entry["selection"].split())+ for entry in _backend_ci()["jobs"]["repo-cpu-tests"]["strategy"]["matrix"]["include"]+ )+++def _shard_pytest_step() -> dict:+ for step in _backend_ci()["jobs"]["repo-cpu-tests"]["steps"]:+ if "${{ matrix.selection }}" in str(step.get("run", "")):+ return step+ raise AssertionError("the sharded pytest step is gone from repo-cpu-tests")diff --git a/tests/test_studio_root_resilience.py b/tests/test_studio_root_resilience.pyindex f16648fc435..edd820c8b2b 100644--- a/tests/test_studio_root_resilience.py+++ b/tests/test_studio_root_resilience.py@@ -16,6 +16,16 @@ STORAGE_ROOTS = REPO_ROOT / "studio" / "backend" / "utils" / "paths" / "storage_LLAMA_CPP = REPO_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py"+# storage_roots.py imports `loggers`, which is studio/backend/loggers. Nothing in this file put+# studio/backend on sys.path, so these tests only passed when a tests/studio module happened to+# have been imported into the same process first -- true while the whole tree ran as one pytest+# session, and false the moment tests/studio runs on its own runner. Two tests then failed with+# ModuleNotFoundError: No module named 'loggers', which names neither this file nor the cause.+_BACKEND = REPO_ROOT / "studio" / "backend"+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++def _load(name: str, path: Path):spec = importlib.util.spec_from_file_location(name, path)assert spec is not None and spec.loader is not None