Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Backend CI: split the fsdp2 Repo tests shard in two (#15851)

Daniel Han06314b97dd65
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 067f926c423..3d528e68fca 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -358,9 +358,19 @@ jobs:
# Native FSDP2's gloo tests spawn 2 to 4 ranks each: about ten minutes of this
# runner on their own, which pushed the catch-all to the 2100 s step cap. They run
# beside it instead.
+ # They then grew to 2092 s of the same cap on their own (364 tests by 2026-10-28)
+ # and were interrupted after passing, so they are split in two by file name: native_d
+ # onwards is about 55% of the measured time, everything else about 45%. Each glob
+ # must match a file, or bash hands pytest the literal pattern
+ # (tests/test_ci_repo_cpu_shards.py checks both).
- shard: fsdp2
selection: >-
- tests/test_fsdp2*.py
+ tests/test_fsdp2_[!n]*.py
+ tests/test_fsdp2_n[!a]*.py
+ tests/test_fsdp2_native_[abc]*.py
+ - shard: fsdp2-native
+ selection: >-
+ tests/test_fsdp2_native_[!abc]*.py
# The other multi-rank gloo suites (tensor, pipeline and context parallel, float8 and
# fp8 under FSDP2) did the same again: with them the catch-all hit the 2100 s cap and was
# interrupted at 74 to 98% on most runs. Locally on 4 cores they are 578 s of the
diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.py
index 12996e913a8..c0af1c0a9e6 100644
--- a/tests/studio/test_backend_ci_parallel_isolation.py
+++ b/tests/studio/test_backend_ci_parallel_isolation.py
@@ -318,15 +318,15 @@ def test_the_command_scan_sees_the_parallel_run_and_the_serial_steps():
"""Pin the parser: a scan that matched nothing would pass both tests above."""
commands = _pytest_commands()
parallel = [command for command in commands if " -n " in f" {command} "]
- assert len(parallel) == 8, (
- f"expected eight parallel pytest runs, three shards of the backend matrix and five "
+ assert len(parallel) == 9, (
+ f"expected nine parallel pytest runs, three shards of the backend matrix and six "
f"of repo-cpu-tests, got {parallel}. If a job stopped running in parallel, or a "
f"shard was added or removed, say so here rather than letting this scan quietly "
f"cover fewer runs."
)
root = [command for command in parallel if _over_the_repo_tests(command)]
- assert len(root) == 5, (
- f"expected the five repo-root shards, got {root}. The isolation checks above apply "
+ assert len(root) == 6, (
+ f"expected the six repo-root shards, got {root}. The isolation checks above apply "
f"to those, and a scan that matched none of them would pass on nothing."
)
backend = [command for command in parallel if _over_the_backend(command)]
diff --git a/tests/test_ci_repo_cpu_shards.py b/tests/test_ci_repo_cpu_shards.py
index 1595236c279..669bf8587ca 100644
--- a/tests/test_ci_repo_cpu_shards.py
+++ b/tests/test_ci_repo_cpu_shards.py
@@ -126,6 +126,26 @@ class TestEveryTestFileLandsInExactlyOneShard:
)
+class TestEveryGlobRootMatchesAFile:
+ def test_no_shard_hands_pytest_an_unexpanded_pattern(self):
+ """bash passes a glob that matches nothing through as the literal pattern, and pytest
+ then fails the whole shard on a path that does not exist. A split that moved the last
+ file out of one of its globs would break that shard rather than drop the file."""
+ files = _test_files()
+ empty = [
+ (name, root)
+ for name, (roots, _) in _shards().items()
+ for root in roots
+ if "*" in root and not any(fnmatch.fnmatch(path, root) for path in files)
+ ]
+ assert not empty, f"these shard globs match no test file: {empty}"
+
+ def test_an_empty_glob_is_reported(self):
+ assert not any(
+ fnmatch.fnmatch(path, "tests/test_no_such_prefix_*.py") for path in _test_files()
+ )
+
+
class TestTheSplitKeepsTheSelectionItInherited:
def test_the_deselected_hub_tests_stay_deselected(self):
"""Both hit huggingface_hub for live model existence checks. They live in a shard-3