Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

ci: run tests/ in parallel with pytest-xdist (~3 min faster) (#3223)

Shimul Debnath82fe41a44eaa
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index a752bf574..63ad1c1a0 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -155,8 +155,9 @@ Open an [issue](https://github.com/debpalash/VoiceStudio/issues/new) with:
2. **Keep PRs focused** — one feature or fix per PR
3. **Run tests** before pushing:
```bash
- # Backend tests (the same two runs as CI)
- uv run pytest tests/ -q
+ # Backend tests (the same two runs as CI; -n auto runs tests/ in parallel,
+ # optional locally, it is about 4x faster on 4 cores)
+ uv run pytest tests/ -q -n auto
uv run pytest backend/tests/ -q
# Frontend build check
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9c296bed7..9d5685e10 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -110,8 +110,12 @@ jobs:
# networked empty-cache run before it was caught). All legitimate HF
# interactions in tests are stubbed; anything that trips this is a
# test-isolation bug.
+ # `-n auto` runs tests/ in parallel (one worker per CPU; 10 min serial ->
+ # ~3 min on a 4-core runner, with identical results). backend/tests/ below
+ # stays serial: one test there depends on another's leftover state and
+ # fails when run in isolation, which parallel scheduling exposes.
- name: Run pytest
- run: uv run --no-sync pytest tests/ -q --tb=short
+ run: uv run --no-sync pytest tests/ -q --tb=short -n auto
env:
HF_HUB_OFFLINE: "1"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3a41bc35..f81ede5d4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -75,6 +75,7 @@ metadata and the backend fallback mirror it.
### CI
+- The main test step runs in parallel with pytest-xdist, about 3 minutes faster per CI run (#3223)
- Interactive release helper: pick stable or beta, draft or publish, and confirm by typing the tag (`scripts/release.py`) (#3187)
- Release helper choices are arrow-key menus (#3190)
- Desktop request-guard self-tests stay deterministic when app requests arrive during cleanup (#2676)
diff --git a/pyproject.toml b/pyproject.toml
index 3d03e57c5..3038a4d16 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -348,6 +348,7 @@ dev = [
"pytest>=9.0.3",
"pytest-asyncio>=1.3.0",
"pytest-cov>=6.0",
+ "pytest-xdist>=3.6",
]
[tool.pytest.ini_options]
diff --git a/tests/conftest.py b/tests/conftest.py
index 011a76c54..2eb2886c8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -21,8 +21,24 @@ if _BACKEND not in sys.path:
# test module, which is what freezes DATA_DIR at `core.config` import time)
# makes every local run behave like a clean CI runner. `setdefault` semantics:
# an explicitly exported OMNIVOICE_DATA_DIR still wins.
-if not os.environ.get("OMNIVOICE_DATA_DIR"):
+#
+# pytest-xdist: the controller imports this file first and exports the dir it
+# made; every worker then inherits it, so all workers shared ONE prefs.json /
+# omnivoice.db / user-env and one worker's `llm_backend=off` flipped another's
+# "ready" assertion. A dir WE created (tracked in _OMNIVOICE_TEST_AUTO_DATA_DIR)
+# is therefore re-made per worker; an explicitly exported one is still honored.
+_AUTO_DATA_DIR = "_OMNIVOICE_TEST_AUTO_DATA_DIR"
+_inherited_auto_dir = os.environ.get("OMNIVOICE_DATA_DIR") if (
+ os.environ.get("PYTEST_XDIST_WORKER")
+ and os.environ.get("OMNIVOICE_DATA_DIR") == os.environ.get(_AUTO_DATA_DIR)
+) else None
+if _inherited_auto_dir:
+ _env_file = os.environ.get("OMNIVOICE_ENV_FILE", "")
+ if _env_file.startswith(_inherited_auto_dir):
+ del os.environ["OMNIVOICE_ENV_FILE"] # re-derived below from the worker's dir
+if not os.environ.get("OMNIVOICE_DATA_DIR") or _inherited_auto_dir:
os.environ["OMNIVOICE_DATA_DIR"] = tempfile.mkdtemp(prefix="omnivoice-test-data-")
+ os.environ[_AUTO_DATA_DIR] = os.environ["OMNIVOICE_DATA_DIR"]
# Same story for the durable per-user env file (~/.config/omnivoice/env):
# `main.py` loads it with override=True at import, so a TestClient importing
# the app mid-suite would inject the developer's real TRANSLATE_* / key vars
diff --git a/tests/test_ci_pytest_parallel.py b/tests/test_ci_pytest_parallel.py
new file mode 100644
index 000000000..03429b3f4
--- /dev/null
+++ b/tests/test_ci_pytest_parallel.py
@@ -0,0 +1,28 @@
+"""CI runs tests/ in parallel (pytest-xdist); the plugin must be a declared dev dependency-group member.
+
+`-n auto` fails with "unrecognized arguments" when the plugin is missing, which
+would break every CI run, so the workflow flag and the dependency move together.
+"""
+from __future__ import annotations
+
+import re
+import tomllib
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_parallel_flag_requires_the_declared_plugin():
+ workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8")
+ uses_parallel = re.search(r"pytest tests/ [^\n]*-n (auto|\d+)", workflow) is not None
+ dev = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))[
+ "dependency-groups"
+ ]["dev"]
+ declared = any(dep.lower().startswith("pytest-xdist") for dep in dev)
+ assert declared or not uses_parallel, "ci.yml uses -n but pytest-xdist is not in the dev dependency group"
+
+
+def test_backend_suite_stays_serial_until_its_order_dependence_is_fixed():
+ workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8")
+ line = next(l for l in workflow.splitlines() if "pytest backend/tests/" in l)
+ assert " -n " not in line
diff --git a/tests/test_mm2_lifecycle.py b/tests/test_mm2_lifecycle.py
index 59d76dfae..02e9357c3 100644
--- a/tests/test_mm2_lifecycle.py
+++ b/tests/test_mm2_lifecycle.py
@@ -110,6 +110,16 @@ def test_omnivoice_unload_idempotent_and_preload_safe():
def test_list_loaded_empty(monkeypatch):
monkeypatch.setattr(mm, "model", None)
monkeypatch.setattr(mm, "_diar_pipeline", None)
+ # list_loaded() also reads these process-wide registries (sidecars, engines,
+ # the active backend, the dictation ASR and the offline translator); an earlier test on
+ # the same pytest-xdist worker can leave an engine or sidecar behind, which
+ # made this test pass serially and fail depending on worker placement.
+ monkeypatch.setattr("services.subprocess_backend.list_live_sidecars", lambda: [])
+ monkeypatch.setattr("services.asr_backend._capture_backend", None)
+ monkeypatch.setattr("api.routers.dub_translate._nllb_model", None)
+ monkeypatch.setattr("api.routers.engines._ENGINE_INSTANCES", {})
+ monkeypatch.setattr("services.tts_backend._active_instance", None)
+ monkeypatch.setattr("services.tts_backend._active_instance_id", None)
out = ml.list_loaded()
assert out == {"models": [], "count": 0} or out["count"] == 0
diff --git a/tests/test_xdist_data_dir_isolation.py b/tests/test_xdist_data_dir_isolation.py
new file mode 100644
index 000000000..962da9df8
--- /dev/null
+++ b/tests/test_xdist_data_dir_isolation.py
@@ -0,0 +1,53 @@
+"""Each pytest-xdist worker must get its own hermetic data dir (tests/conftest.py).
+
+The controller exports the dir it created and workers inherit the environment,
+so before the fix all workers shared one prefs.json/omnivoice.db. That made the
+LLM skill tests fail in parallel CI runs (#3223) while passing serially.
+"""
+from __future__ import annotations
+
+import importlib.util
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+_PROBE_OUT = os.environ.get("XDIST_DATA_DIR_PROBE_OUT")
+
+
[email protected](not _PROBE_OUT, reason="driven by test_each_xdist_worker_gets_its_own_data_dir")
[email protected]("i", range(12))
+def test_probe_records_worker_and_data_dir(i):
+ with open(_PROBE_OUT, "a", encoding="utf-8") as out:
+ data_dir = os.environ["OMNIVOICE_DATA_DIR"]
+ env_file = os.environ["OMNIVOICE_ENV_FILE"]
+ # Directory names only (mkdtemp names are unique), never absolute paths.
+ out.write(
+ f"{os.environ.get('PYTEST_XDIST_WORKER')}\t{os.path.basename(data_dir)}"
+ f"\t{int(os.path.dirname(env_file) == data_dir)}\n"
+ )
+
+
[email protected](importlib.util.find_spec("xdist") is None, reason="pytest-xdist not installed")
+def test_each_xdist_worker_gets_its_own_data_dir(tmp_path):
+ out = tmp_path / "probe.tsv"
+ env = {k: v for k, v in os.environ.items() if not k.startswith(("OMNIVOICE_DATA_DIR", "OMNIVOICE_ENV_FILE", "_OMNIVOICE_TEST"))}
+ env["XDIST_DATA_DIR_PROBE_OUT"] = str(out)
+ subprocess.run(
+ [sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-q", "-n", "3",
+ f"{Path(__file__)}::test_probe_records_worker_and_data_dir"],
+ cwd=ROOT, env=env, check=True, capture_output=True, timeout=120,
+ )
+ rows = [line.split("\t") for line in out.read_text(encoding="utf-8").splitlines()]
+ by_worker = {}
+ for worker, data_dir, env_in_data_dir in rows:
+ by_worker.setdefault(worker, set()).add((data_dir, env_in_data_dir))
+ assert len(by_worker) >= 2, "xdist ran everything on one worker; nothing to compare"
+ assert all(len(v) == 1 for v in by_worker.values()), "a worker changed its data dir mid-run"
+ dirs = [next(iter(v))[0] for v in by_worker.values()]
+ assert len(set(dirs)) == len(dirs), f"workers share a data dir: {dirs}"
+ for _, env_in_data_dir in (next(iter(v)) for v in by_worker.values()):
+ assert env_in_data_dir == "1", "user-env file must live in the worker's own dir"
diff --git a/uv.lock b/uv.lock
index 932852eba..1e082d22a 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1718,6 +1718,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/29/64/0b75bc50ec53b4e000bac913625511215aa96124adf5dba8c4baa17c02cd/espeakng_loader-0.2.4-py3-none-win_arm64.whl", hash = "sha256:d7a2928843eaeb2df82f99a370f44e8a630f59b02f9b0d1f168a03c4eeb76b89", size = 9426841, upload-time = "2025-01-17T01:23:21.766Z" },
]
+[[package]]
+name = "execnet"
+version = "2.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
+]
+
[[package]]
name = "fastapi"
version = "0.136.3"
@@ -4245,6 +4254,7 @@ dev = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-cov" },
+ { name = "pytest-xdist" },
]
[package.metadata]
@@ -4340,6 +4350,7 @@ dev = [
{ name = "pytest", specifier = ">=9.0.3" },
{ name = "pytest-asyncio", specifier = ">=1.3.0" },
{ name = "pytest-cov", specifier = ">=6.0" },
+ { name = "pytest-xdist", specifier = ">=3.6" },
]
[[package]]
@@ -5428,6 +5439,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
]
+[[package]]
+name = "pytest-xdist"
+version = "3.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "execnet" },
+ { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
+]
+
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"