Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Backend CI: run the context parallel compose gloo suite in its own shard (#16447)
Daniel Hana8480378aba3
diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.ymlindex 1576f4ffb5b..405899f54f2 100644--- a/.github/workflows/studio-backend-ci.yml+++ b/.github/workflows/studio-backend-ci.yml@@ -398,8 +398,20 @@ jobs:selection: >-tests/test_[!fpt]*gloo*.pytests/test_f[!s]*gloo*.py- tests/test_p[!p]*gloo*.py+ tests/test_parallel_[!c]*gloo*.pytests/test_tp_[!n]*gloo*.py+ # #16407 grew test_parallel_cp_compose_gloo.py to about a third of the gloo shard's time+ # (1143 of 3605 test-seconds measured per file), which took the shard to 1922 s of the+ # 2100 s cap on main and over it on a slower runner. gloo-tp-pp has no room for it either+ # (1397 s), so test_parallel_c*gloo* runs on its own. A *gloo* file named test_p[!ap]*+ # would match no shard; tests/test_ci_repo_cpu_shards.py reports it as dropped.+ # Its tests spawn 2 to 8 gloo ranks each: four at once peaked at about 20 GB (summed PSS,+ # measured locally) and the 16 GB runner was shut down mid-run, twice. Two at once peak+ # at about 12 GB.+ - shard: gloo-cp+ workers: 2+ selection: >-+ tests/test_parallel_c*gloo*.py# The gloo shard then ran 1331 to 2058 s of the same cap on 2026-10-29. Measured per# file, native tensor parallel plus pipeline parallel are about 51% of it, everything# else about 49%. A *gloo* file named test_t[!p]* or test_tp[!_]* would match neither@@ -481,7 +493,7 @@ jobs:# globals. The outer timeout must fire before the job cap.run: |timeout --signal=INT --kill-after=60 2100 \- python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n 4 --dist loadgroup --timeout=330 \+ python -m pytest ${{ matrix.selection }} -q -rs --tb=short -n ${{ matrix.workers || 4 }} --dist loadgroup --timeout=330 \-m 'not server and not e2e'- name: Hardware-spoof tests (state-sensitive, run in isolation)diff --git a/tests/studio/test_backend_ci_parallel_isolation.py b/tests/studio/test_backend_ci_parallel_isolation.pyindex de917cc4183..68f655a2a09 100644--- a/tests/studio/test_backend_ci_parallel_isolation.py+++ b/tests/studio/test_backend_ci_parallel_isolation.py@@ -28,18 +28,30 @@ def _jobs() -> dict:return yaml.safe_load(WORKFLOW.read_text(encoding = "utf-8"))["jobs"]-def _selections(job_name: str) -> list[str]:+def _entries(job_name: str) -> list[dict]:"""Entries without a `selection` are skipped, since the floor spot-check leg names its filesdirectly."""job = _jobs()[job_name]include = job.get("strategy", {}).get("matrix", {}).get("include", [])- return [" ".join(entry["selection"].split()) for entry in include if "selection" in entry]+ return [entry for entry in include if "selection" in entry]+++def _selections(job_name: str) -> list[str]:+ return [" ".join(entry["selection"].split()) for entry in _entries(job_name)]+++_WORKERS = re.compile(r"\$\{\{ matrix\.workers \|\| (\d+) \}\}")+++def _expand(line: str, entry: dict) -> str:+ """One shard's command: its selection, and its `workers` or the expression's default."""+ line = line.replace("${{ matrix.selection }}", " ".join(entry["selection"].split()))+ return _WORKERS.sub(lambda match: str(entry.get("workers", match.group(1))), line)def _commands_in(job_name: str) -> list[str]:"""Expands `${{ matrix.selection }}` from this job's own matrix, not the other parallel job's."""job = _jobs()[job_name]- selections = _selections(job_name)commands = []for step in job.get("steps", []):joined = re.sub(r"\\\s*\n\s*", " ", str(step.get("run", "")))@@ -48,9 +60,7 @@ def _commands_in(job_name: str) -> list[str]:if "python -m pytest" not in line or line.startswith("#"):continueif "${{ matrix.selection }}" in line:- commands.extend(- line.replace("${{ matrix.selection }}", selection) for selection in selections- )+ commands.extend(_expand(line, entry) for entry in _entries(job_name))else:commands.append(line)return commands@@ -328,15 +338,15 @@ def test_the_command_scan_sees_the_parallel_run_and_the_serial_steps():"""Pin the parser: a scan that matched nothing would pass both tests above."""commands = _pytest_commands()parallel = [command for command in commands if " -n " in f" {command} "]- assert len(parallel) == 11, (- f"expected eleven parallel pytest runs, three shards of the backend matrix and eight "+ assert len(parallel) == 12, (+ f"expected twelve parallel pytest runs, three shards of the backend matrix and nine "f"of repo-cpu-tests, got {parallel}. If a job stopped running in parallel, or a "f"shard was added or removed, say so here rather than letting this scan quietly "f"cover fewer runs.")root = [command for command in parallel if _over_the_repo_tests(command)]- assert len(root) == 8, (- f"expected the eight repo-root shards, got {root}. The isolation checks above apply "+ assert len(root) == 9, (+ f"expected the nine repo-root shards, got {root}. The isolation checks above apply "f"to those, and a scan that matched none of them would pass on nothing.")backend = [command for command in parallel if _over_the_backend(command)]