Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Backend CI: run the context parallel compose gloo suite in its own shard (#16447)

Daniel Hana8480378aba3
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml
index 1576f4ffb5b..405899f54f2 100644
--- a/.github/workflows/studio-backend-ci.yml
+++ b/.github/workflows/studio-backend-ci.yml
@@ -398,8 +398,20 @@ jobs:
selection: >-
tests/test_[!fpt]*gloo*.py
tests/test_f[!s]*gloo*.py
- tests/test_p[!p]*gloo*.py
+ tests/test_parallel_[!c]*gloo*.py
tests/test_tp_[!n]*gloo*.py
+ # #16407 grew test_parallel_cp_compose_gloo.py to about a third of the gloo shard's time
+ # (1143 of 3605 test-seconds measured per file), which took the shard to 1922 s of the
+ # 2100 s cap on main and over it on a slower runner. gloo-tp-pp has no room for it either
+ # (1397 s), so test_parallel_c*gloo* runs on its own. A *gloo* file named test_p[!ap]*
+ # would match no shard; tests/test_ci_repo_cpu_shards.py reports it as dropped.
+ # Its tests spawn 2 to 8 gloo ranks each: four at once peaked at about 20 GB (summed PSS,
+ # measured locally) and the 16 GB runner was shut down mid-run, twice. Two at once peak
+ # at about 12 GB.
+ - shard: gloo-cp
+ workers: 2
+ selection: >-
+ tests/test_parallel_c*gloo*.py
# The gloo shard then ran 1331 to 2058 s of the same cap on 2026-10-29. Measured per
# file, native tensor parallel plus pipeline parallel are about 51% of it, everything
# else about 49%. A *gloo* file named test_t[!p]* or test_tp[!_]* would match neither
@@ -481,7 +493,7 @@ jobs:
# globals. The outer timeout must fire before the job cap.
run: |
timeout --signal=INT --kill-after=60 2100 \
- python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n 4 --dist loadgroup --timeout=330 \
+ python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n ${{ matrix.workers || 4 }} --dist loadgroup --timeout=330 \
-m 'not server and not e2e'
- name: Hardware-spoof tests (state-sensitive, run in isolation)
diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.py
index de917cc4183..68f655a2a09 100644
--- a/tests/studio/test_backend_ci_parallel_isolation.py
+++ b/tests/studio/test_backend_ci_parallel_isolation.py
@@ -28,18 +28,30 @@ def _jobs() -> dict:
return yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))["jobs"]
-def _selections(job_name: str) -> list[str]:
+def _entries(job_name: str) -> list[dict]:
"""Entries without a `selection` are skipped, since the floor spot-check leg names its files
directly."""
job = _jobs()[job_name]
include = job.get("strategy", {}).get("matrix", {}).get("include", [])
- return [" ".join(entry["selection"].split()) for entry in include if "selection" in entry]
+ return [entry for entry in include if "selection" in entry]
+
+
+def _selections(job_name: str) -> list[str]:
+ return [" ".join(entry["selection"].split()) for entry in _entries(job_name)]
+
+
+_WORKERS = re.compile(r"\$\{\{ matrix\.workers \|\| (\d+) \}\}")
+
+
+def _expand(line: str, entry: dict) -> str:
+ """One shard's command: its selection, and its `workers` or the expression's default."""
+ line = line.replace("${{ matrix.selection }}", " ".join(entry["selection"].split()))
+ return _WORKERS.sub(lambda match: str(entry.get("workers", match.group(1))), line)
def _commands_in(job_name: str) -> list[str]:
"""Expands `${{ matrix.selection }}` from this job's own matrix, not the other parallel job's."""
job = _jobs()[job_name]
- selections = _selections(job_name)
commands = []
for step in job.get("steps", []):
joined = re.sub(r"\\\s*\n\s*", " ", str(step.get("run", "")))
@@ -48,9 +60,7 @@ def _commands_in(job_name: str) -> list[str]:
if "python -m pytest" not in line or line.startswith("#"):
continue
if "${{ matrix.selection }}" in line:
- commands.extend(
- line.replace("${{ matrix.selection }}", selection) for selection in selections
- )
+ commands.extend(_expand(line, entry) for entry in _entries(job_name))
else:
commands.append(line)
return commands
@@ -328,15 +338,15 @@ def test_the_command_scan_sees_the_parallel_run_and_the_serial_steps():
"""Pin the parser: a scan that matched nothing would pass both tests above."""
commands = _pytest_commands()
parallel = [command for command in commands if " -n " in f" {command} "]
- assert len(parallel) == 11, (
- f"expected eleven parallel pytest runs, three shards of the backend matrix and eight "
+ assert len(parallel) == 12, (
+ f"expected twelve parallel pytest runs, three shards of the backend matrix and nine "
f"of repo-cpu-tests, got {parallel}. If a job stopped running in parallel, or a "
f"shard was added or removed, say so here rather than letting this scan quietly "
f"cover fewer runs."
)
root = [command for command in parallel if _over_the_repo_tests(command)]
- assert len(root) == 8, (
- f"expected the eight repo-root shards, got {root}. The isolation checks above apply "
+ assert len(root) == 9, (
+ f"expected the nine repo-root shards, got {root}. The isolation checks above apply "
f"to those, and a scan that matched none of them would pass on nothing."
)
backend = [command for command in parallel if _over_the_backend(command)]