Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
CI: run the 34 mocked GPU-selection tests, drop dead zoo deselects, print skip reasons (#13706)
Daniel Han3a6179b4a860
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.ymlindex e0cf0045f32..3aff9cf156c 100644--- a/.github/workflows/consolidated-tests-ci.yml+++ b/.github/workflows/consolidated-tests-ci.yml@@ -2389,17 +2389,14 @@ jobs:unsloth-zoo-ref: ${{ env.UNSLOTH_ZOO_REF }}- name: unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} — full pytest (CPU)- # 106 of 111 test_* in unsloth_zoo are CPU-only. The two CUDA-skip- # cases below auto-skip on a GPU-less runner; deselect them- # explicitly so the no-CUDA outcome is "deselected", not "skipped",- # making intent visible in the report. Env inherited from job block.+ # 106 of 111 test_* in unsloth_zoo are CPU-only. The CUDA-skip case+ # below auto-skips on a GPU-less runner; deselect it explicitly so the+ # no-CUDA outcome is "deselected", not "skipped", making intent+ # visible in the report. Env inherited from job block.#- # test_get_peft_model_passes_finetune_last_n_layers_through is- # deselected because unsloth_zoo/mlx/loader.py at line 2972 calls- # model.trainable_parameters() on the fake-model fixture, which- # the test never stubbed; this fails on every platform regardless- # of CUDA. Tracked upstream as an unsloth_zoo bug; deselecting- # here unblocks unsloth CI until the loader fixture is fixed.+ # -rs prints every skip reason, so a dependency that goes missing and+ # turns a module into a skip shows up in the log instead of only in+ # the skipped count.## A test that hangs rather than fails used to consume the whole job# budget and report nothing usable: the job ended as "cancelled", with@@ -2428,12 +2425,11 @@ jobs:# so exclude that family from xdist and run it separately below. The other# ignores either leak process state or use timing assertions that need a quiet# process.- python -m pytest -q --tb=short -n 4 --dist loadfile tests/ \+ python -m pytest -q -rs --tb=short -n 4 --dist loadfile tests/ \--ignore-glob='tests/test_mlx_*.py' \--ignore=tests/test_moe_bnb4bit_per_expert_conversions.py \--ignore=tests/test_hf_xet_fallback.py \--ignore=tests/test_gemma3_forced_float32_boundary_dtype.py \- --deselect tests/test_unsloth_zoo_lora_merge.py::test_active_merge_device_returns_string_on_cuda_host \--deselect tests/test_unsloth_zoo_lora_merge.py::test_merge_lora_moves_cpu_inputs_to_active_device# Run contamination-sensitive files in separate processes. Keep collecting@@ -2465,9 +2461,7 @@ jobs:echo "::error::the mlx group glob matched no files"exit 1fi- python -m pytest -q --tb=short $mlx_group \- --deselect tests/test_mlx_finetune_last_n_layers.py::test_get_peft_model_passes_finetune_last_n_layers_through \- || _keep "$?" "the mlx group"+ python -m pytest -q -rs --tb=short $mlx_group || _keep "$?" "the mlx group"exit "$rc"- name: unsloth_zoo — test_apply_fused_lm_head (lives in compiler.py)@@ -2513,7 +2507,7 @@ jobs:pip show unsloth_zooecho "::endgroup::"echo "Core zoo job done. Coverage:"- echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)"+ echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (GPU-only cases deselected)"echo " - unsloth_zoo.compiler.test_apply_fused_lm_head"- name: Save the pip cachediff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.ymlindex 7eaca683fe4..cc16ad709c0 100644--- a/.github/workflows/studio-backend-ci.yml+++ b/.github/workflows/studio-backend-ci.yml@@ -255,7 +255,8 @@ jobs:- name: Backend testsif: matrix.scope == 'full'working-directory: studio/backend- # Deselections need a real GPU or a live llama.cpp process.+ # The one deselection needs a live llama.cpp process. The GPU-selection, KV-cache and+ # CUDA-util tests mock torch.cuda and the device probe, so they run here on CPU torch.## -n 4 verified order-independent (same failure set as serial).#@@ -267,7 +268,7 @@ jobs:# Shared selection lives here so every shard inherits one list; shards differ only in matrix.selection.run: |timeout --signal=INT --kill-after=60 1500 \- python -m pytest ${{ matrix.selection }} -q --tb=short -n 4 --dist loadgroup --timeout=330 \+ python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n 4 --dist loadgroup --timeout=330 \--ignore=tests/test_studio_api.py \--ignore=tests/test_streaming_stripper.py \--ignore=tests/test_llama_cpp_wait_for_vram_settle.py \@@ -282,7 +283,7 @@ jobs:--ignore=tests/test_profile_stats.py \--ignore=tests/test_media_auto_switch.py \--ignore=tests/test_pr5624_regressions.py \- -k 'not llama_cpp_load_progress_live and not TestGpuAutoSelection and not TestPreSpawnGpuResolution and not TestPerGpuFitGuardAllCounts and not TestTransformersIntrospection and not test_returns_cuda_when_cuda_available and not test_calls_cuda_cache_when_cuda'+ -k 'not llama_cpp_load_progress_live'- name: Backend tests that cannot share a worker# The twelve serial files, one process.@@ -551,7 +552,7 @@ jobs:# Timeouts mirror the backend pytest job; the outer one must fire before the job cap.run: |timeout --signal=INT --kill-after=60 2100 \- python -m pytest ${{ matrix.selection }} -q --tb=short -n 4 --dist loadgroup --timeout=330 \+ python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n 4 --dist loadgroup --timeout=330 \-m 'not server and not e2e'- name: Hardware-spoof tests (state-sensitive, run in isolation)diff --git a/tests/studio/test_zoo_suite_parallel_isolation.py b/tests/studio/test_zoo_suite_parallel_isolation.pyindex 1d59e9b07ec..0a9374a6a2e 100644--- a/tests/studio/test_zoo_suite_parallel_isolation.py+++ b/tests/studio/test_zoo_suite_parallel_isolation.py@@ -57,8 +57,9 @@ ISOLATED = [ZOO_MARKER = "--dist loadfile tests/"-# Deselected because it needs a GPU.-MLX_DESELECT = (+# Was deselected while the zoo fixture did not stub model.trainable_parameters(). The zoo+# fixed the fixture, and it passes on CPU in the serial mlx group, so it must run there.+MLX_FORMERLY_DESELECTED = ("tests/test_mlx_finetune_last_n_layers.py::""test_get_peft_model_passes_finetune_last_n_layers_through")@@ -166,12 +167,17 @@ def test_the_deselects_survive_on_the_parallel_run() -> None:"""Keep each deselect with the command that owns its file."""cmd = _zoo_parallel()assert (- cmd.count("--deselect") == 2- ), f"the parallel zoo run carries {cmd.count('--deselect')} deselects, expected 2"+ cmd.count("--deselect") == 1+ ), f"the parallel zoo run carries {cmd.count('--deselect')} deselects, expected 1"+ assert "test_merge_lora_moves_cpu_inputs_to_active_device" in cmd+++def test_the_mlx_group_runs_the_last_n_layers_test() -> None:+ """A deselect of a test that passes hides it; this one passes on CPU since the zoo fix."""group = _zoo_mlx_group()- assert MLX_DESELECT in group, (- f"{MLX_DESELECT} is deselected nowhere now that test_mlx_finetune_last_n_layers.py "- f"runs in the serial mlx group, so it fails on a GPU-less runner instead"+ assert MLX_FORMERLY_DESELECTED not in group, (+ f"{MLX_FORMERLY_DESELECTED} is deselected again, but it passes in the serial mlx "+ f"group on a GPU-less runner")diff --git a/tests/test_ci_backend_pytest_shards.py b/tests/test_ci_backend_pytest_shards.pyindex 0b3fd9d78c7..0ec1898114f 100644--- a/tests/test_ci_backend_pytest_shards.py+++ b/tests/test_ci_backend_pytest_shards.py@@ -292,20 +292,30 @@ class TestEveryTestFileLandsInExactlyOneShard:class TestTheSplitKeepsTheSelectionItInherited:def test_every_shard_keeps_the_marker_filter(self):- """The environment-specific deselections: live GPU introspection and a real- llama.cpp process, none of which exist on a GPU-less runner. The filter is on the- shared step rather than per shard, so it is checked there."""+ """The one environment-specific deselection: a real llama.cpp process, which a+ GPU-less runner does not have. The filter is on the shared step rather than per+ shard, so it is checked there."""run = _shared_pytest_step()["run"]- for fragment in (- "not llama_cpp_load_progress_live",- "not TestGpuAutoSelection",- "not TestPreSpawnGpuResolution",- "not TestPerGpuFitGuardAllCounts",- "not TestTransformersIntrospection",- "not test_returns_cuda_when_cuda_available",- "not test_calls_cuda_cache_when_cuda",+ assert (+ "not llama_cpp_load_progress_live" in run+ ), "the -k filter lost the live llama.cpp test"++ def test_mocked_gpu_tests_are_not_deselected(self):+ """These classes patch torch.cuda and the device probe, so they pass on CPU torch.+ They were deselected as "needs a real GPU" from May to October 2026 and ran nowhere;+ a deselection of them here would hide them again."""+ run = _shared_pytest_step()["run"]+ for name in (+ "TestGpuAutoSelection",+ "TestPreSpawnGpuResolution",+ "TestPerGpuFitGuardAllCounts",+ "TestTransformersIntrospection",+ "test_returns_cuda_when_cuda_available",+ "test_calls_cuda_cache_when_cuda",):- assert fragment in run, f"the -k filter lost {fragment!r}"+ assert (+ f"not {name}" not in run+ ), f"{name} is mocked and must run here, not be deselected"def test_every_shard_keeps_the_timeouts(self):"""A shard that loses these reports "cancelled" and names nothing, which is what