Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CI: run the 34 mocked GPU-selection tests, drop dead zoo deselects, print skip reasons (#13706)

Daniel Han3a6179b4a860
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml
index e0cf0045f32..3aff9cf156c 100644
--- a/.github/workflows/consolidated-tests-ci.yml
+++ b/.github/workflows/consolidated-tests-ci.yml
@@ -2389,17 +2389,14 @@ jobs:
unsloth-zoo-ref: ${{ env.UNSLOTH_ZOO_REF }}
- name: unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} — full pytest (CPU)
- # 106 of 111 test_* in unsloth_zoo are CPU-only. The two CUDA-skip
- # cases below auto-skip on a GPU-less runner; deselect them
- # explicitly so the no-CUDA outcome is "deselected", not "skipped",
- # making intent visible in the report. Env inherited from job block.
+ # 106 of 111 test_* in unsloth_zoo are CPU-only. The CUDA-skip case
+ # below auto-skips on a GPU-less runner; deselect it explicitly so the
+ # no-CUDA outcome is "deselected", not "skipped", making intent
+ # visible in the report. Env inherited from job block.
#
- # test_get_peft_model_passes_finetune_last_n_layers_through is
- # deselected because unsloth_zoo/mlx/loader.py at line 2972 calls
- # model.trainable_parameters() on the fake-model fixture, which
- # the test never stubbed; this fails on every platform regardless
- # of CUDA. Tracked upstream as an unsloth_zoo bug; deselecting
- # here unblocks unsloth CI until the loader fixture is fixed.
+ # -rs prints every skip reason, so a dependency that goes missing and
+ # turns a module into a skip shows up in the log instead of only in
+ # the skipped count.
#
# A test that hangs rather than fails used to consume the whole job
# budget and report nothing usable: the job ended as "cancelled", with
@@ -2428,12 +2425,11 @@ jobs:
# so exclude that family from xdist and run it separately below. The other
# ignores either leak process state or use timing assertions that need a quiet
# process.
- python -m pytest -q --tb=short -n 4 --dist loadfile tests/ \
+ python -m pytest -q -rs --tb=short -n 4 --dist loadfile tests/ \
--ignore-glob='tests/test_mlx_*.py' \
--ignore=tests/test_moe_bnb4bit_per_expert_conversions.py \
--ignore=tests/test_hf_xet_fallback.py \
--ignore=tests/test_gemma3_forced_float32_boundary_dtype.py \
- --deselect tests/test_unsloth_zoo_lora_merge.py::test_active_merge_device_returns_string_on_cuda_host \
--deselect tests/test_unsloth_zoo_lora_merge.py::test_merge_lora_moves_cpu_inputs_to_active_device
# Run contamination-sensitive files in separate processes. Keep collecting
@@ -2465,9 +2461,7 @@ jobs:
echo "::error::the mlx group glob matched no files"
exit 1
fi
- python -m pytest -q --tb=short $mlx_group \
- --deselect tests/test_mlx_finetune_last_n_layers.py::test_get_peft_model_passes_finetune_last_n_layers_through \
- || _keep "$?" "the mlx group"
+ python -m pytest -q -rs --tb=short $mlx_group || _keep "$?" "the mlx group"
exit "$rc"
- name: unsloth_zoo — test_apply_fused_lm_head (lives in compiler.py)
@@ -2513,7 +2507,7 @@ jobs:
pip show unsloth_zoo
echo "::endgroup::"
echo "Core zoo job done. Coverage:"
- echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)"
+ echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (GPU-only cases deselected)"
echo " - unsloth_zoo.compiler.test_apply_fused_lm_head"
- name: Save the pip cache
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 7eaca683fe4..cc16ad709c0 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -255,7 +255,8 @@ jobs:
- name: Backend tests
if: matrix.scope == 'full'
working-directory: studio/backend
- # Deselections need a real GPU or a live llama.cpp process.
+ # The one deselection needs a live llama.cpp process. The GPU-selection, KV-cache and
+ # CUDA-util tests mock torch.cuda and the device probe, so they run here on CPU torch.
#
# -n 4 verified order-independent (same failure set as serial).
#
@@ -267,7 +268,7 @@ jobs:
# Shared selection lives here so every shard inherits one list; shards differ only in matrix.selection.
run: |
timeout --signal=INT --kill-after=60 1500 \
- python -m pytest ${{ matrix.selection }} -q --tb=short -n 4 --dist loadgroup --timeout=330 \
+ python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n 4 --dist loadgroup --timeout=330 \
--ignore=tests/test_studio_api.py \
--ignore=tests/test_streaming_stripper.py \
--ignore=tests/test_llama_cpp_wait_for_vram_settle.py \
@@ -282,7 +283,7 @@ jobs:
--ignore=tests/test_profile_stats.py \
--ignore=tests/test_media_auto_switch.py \
--ignore=tests/test_pr5624_regressions.py \
- -k 'not llama_cpp_load_progress_live and not TestGpuAutoSelection and not TestPreSpawnGpuResolution and not TestPerGpuFitGuardAllCounts and not TestTransformersIntrospection and not test_returns_cuda_when_cuda_available and not test_calls_cuda_cache_when_cuda'
+ -k 'not llama_cpp_load_progress_live'
- name: Backend tests that cannot share a worker
# The twelve serial files, one process.
@@ -551,7 +552,7 @@ jobs:
# Timeouts mirror the backend pytest job; the outer one must fire before the job cap.
run: |
timeout --signal=INT --kill-after=60 2100 \
- python -m pytest ${{ matrix.selection }} -q --tb=short -n 4 --dist loadgroup --timeout=330 \
+ python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n 4 --dist loadgroup --timeout=330 \
-m 'not server and not e2e'
- name: Hardware-spoof tests (state-sensitive, run in isolation)
diff --git a/tests/studio/test_zoo_suite_parallel_isolation.py b/tests/studio/test_zoo_suite_parallel_isolation.py
index 1d59e9b07ec..0a9374a6a2e 100644
--- a/tests/studio/test_zoo_suite_parallel_isolation.py
+++ b/tests/studio/test_zoo_suite_parallel_isolation.py
@@ -57,8 +57,9 @@ ISOLATED = [
ZOO_MARKER = "--dist loadfile tests/"
-# Deselected because it needs a GPU.
-MLX_DESELECT = (
+# Was deselected while the zoo fixture did not stub model.trainable_parameters(). The zoo
+# fixed the fixture, and it passes on CPU in the serial mlx group, so it must run there.
+MLX_FORMERLY_DESELECTED = (
"tests/test_mlx_finetune_last_n_layers.py::"
"test_get_peft_model_passes_finetune_last_n_layers_through"
)
@@ -166,12 +167,17 @@ def test_the_deselects_survive_on_the_parallel_run() -> None:
"""Keep each deselect with the command that owns its file."""
cmd = _zoo_parallel()
assert (
- cmd.count("--deselect") == 2
- ), f"the parallel zoo run carries {cmd.count('--deselect')} deselects, expected 2"
+ cmd.count("--deselect") == 1
+ ), f"the parallel zoo run carries {cmd.count('--deselect')} deselects, expected 1"
+ assert "test_merge_lora_moves_cpu_inputs_to_active_device" in cmd
+
+
+def test_the_mlx_group_runs_the_last_n_layers_test() -> None:
+ """A deselect of a test that passes hides it; this one passes on CPU since the zoo fix."""
group = _zoo_mlx_group()
- assert MLX_DESELECT in group, (
- f"{MLX_DESELECT} is deselected nowhere now that test_mlx_finetune_last_n_layers.py "
- f"runs in the serial mlx group, so it fails on a GPU-less runner instead"
+ assert MLX_FORMERLY_DESELECTED not in group, (
+ f"{MLX_FORMERLY_DESELECTED} is deselected again, but it passes in the serial mlx "
+ f"group on a GPU-less runner"
)
diff --git a/tests/test_ci_backend_pytest_shards.py b/tests/test_ci_backend_pytest_shards.py
index 0b3fd9d78c7..0ec1898114f 100644
--- a/tests/test_ci_backend_pytest_shards.py
+++ b/tests/test_ci_backend_pytest_shards.py
@@ -292,20 +292,30 @@ class TestEveryTestFileLandsInExactlyOneShard:
class TestTheSplitKeepsTheSelectionItInherited:
def test_every_shard_keeps_the_marker_filter(self):
- """The environment-specific deselections: live GPU introspection and a real
- llama.cpp process, none of which exist on a GPU-less runner. The filter is on the
- shared step rather than per shard, so it is checked there."""
+ """The one environment-specific deselection: a real llama.cpp process, which a
+ GPU-less runner does not have. The filter is on the shared step rather than per
+ shard, so it is checked there."""
run = _shared_pytest_step()["run"]
- for fragment in (
- "not llama_cpp_load_progress_live",
- "not TestGpuAutoSelection",
- "not TestPreSpawnGpuResolution",
- "not TestPerGpuFitGuardAllCounts",
- "not TestTransformersIntrospection",
- "not test_returns_cuda_when_cuda_available",
- "not test_calls_cuda_cache_when_cuda",
+ assert (
+ "not llama_cpp_load_progress_live" in run
+ ), "the -k filter lost the live llama.cpp test"
+
+ def test_mocked_gpu_tests_are_not_deselected(self):
+ """These classes patch torch.cuda and the device probe, so they pass on CPU torch.
+ They were deselected as "needs a real GPU" from May to October 2026 and ran nowhere;
+ a deselection of them here would hide them again."""
+ run = _shared_pytest_step()["run"]
+ for name in (
+ "TestGpuAutoSelection",
+ "TestPreSpawnGpuResolution",
+ "TestPerGpuFitGuardAllCounts",
+ "TestTransformersIntrospection",
+ "test_returns_cuda_when_cuda_available",
+ "test_calls_cuda_cache_when_cuda",
):
- assert fragment in run, f"the -k filter lost {fragment!r}"
+ assert (
+ f"not {name}" not in run
+ ), f"{name} is mocked and must run here, not be deselected"
def test_every_shard_keeps_the_timeouts(self):
"""A shard that loses these reports "cancelled" and names nothing, which is what