DSPy: The framework for programming—not prompting—language models
Python
ci(tests): parallelize pytest with xdist and simplify CI jobs (#10078)
Isaac Miller6abd518a2b48
diff --git a/.github/workflows/dependency-range.yml b/.github/workflows/dependency-range.ymlindex 8cac7606f..3b15a6119 100644--- a/.github/workflows/dependency-range.yml+++ b/.github/workflows/dependency-range.yml@@ -13,6 +13,10 @@ on:permissions: {}+concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}+jobs:dependency-range:name: Dependencies (${{ matrix.profile }})@@ -84,7 +88,7 @@ jobs:PROFILE: ${{ matrix.profile }}run: |set +e- uv run --frozen --no-sync pytest -q tests/ --extra --deno 2>&1 \+ uv run --frozen --no-sync pytest -q -n auto --dist worksteal tests/ --extra --deno 2>&1 \| tee "$RUNNER_TEMP/pytest-$PROFILE.log"TEST_STATUS=${PIPESTATUS[0]}set -ediff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.ymlindex 00898b716..49396ffb2 100644--- a/.github/workflows/run_tests.yml+++ b/.github/workflows/run_tests.yml@@ -9,6 +9,10 @@ on:permissions: {}+concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}+jobs:workflow-lint:name: Lint GitHub Actions Workflows@@ -99,16 +103,12 @@ jobs:run: |uv sync --dev -p .venv --extra devuv pip list- - name: Run lint with tests- uses: chartboost/ruff-action@e18ae971ccee1b2d7bbef113930f00c670b78da4 # v1- with:- args: check --fix-only- name: Run tests with pytest- run: uv run -p .venv pytest -vv tests/+ run: uv run -p .venv pytest -vv -n auto --dist worksteal tests/- name: Install optional dependenciesrun: uv sync -p .venv --extra dev --extra test_extras- name: Run extra and deno tests- run: uv run -p .venv pytest tests/ -m 'extra or deno' --extra --deno+ run: uv run -p .venv pytest tests/ -m 'extra or deno' --extra --deno -n auto --dist workstealllm_call_test:name: Run Tests with Real LM@@ -201,15 +201,11 @@ jobs:cache-dependency-glob: |**/pyproject.toml**/uv.lock- - name: Create and activate virtual environment- run: |- uv venv .venv- echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"- - name: Install dependencies- run: uv sync --dev -p .venv --extra dev- name: Build- run: uv run -p .venv python -m build+ run: uv build- name: Install built package- run: uv pip install dist/*.whl -p .venv+ run: |+ uv venv .venv+ uv pip install dist/*.whl -p .venv- name: Test import dspy- run: uv run -p .venv python -c "import dspy"+ run: .venv/bin/python -c "import dspy"diff --git a/pyproject.toml b/pyproject.tomlindex 18240fd40..39353ed93 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -49,6 +49,7 @@ dev = ["pytest>=6.2.5","pytest-mock>=3.12.0","pytest-asyncio>=0.26.0",+ "pytest-xdist>=3.5.0","ruff>=0.3.0","pre-commit>=3.7.0","pillow>=10.1.0",diff --git a/tests/test_utils/server/__init__.py b/tests/test_utils/server/__init__.pyindex 75535a1d5..dbc735e9e 100644--- a/tests/test_utils/server/__init__.py+++ b/tests/test_utils/server/__init__.py@@ -76,7 +76,7 @@ def _get_random_port():return s.getsockname()[1]-def _wait_for_port(host, port, timeout=10):+def _wait_for_port(host, port, timeout=60):start_time = time.time()while time.time() - start_time < timeout:with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:diff --git a/uv.lock b/uv.lockindex ac1eb4042..07d5de5d3 100644--- a/uv.lock+++ b/uv.lock@@ -761,7 +761,7 @@ wheels = [[[package]]name = "dspy"-version = "3.2.1"+version = "3.3.0b1"source = { editable = "." }dependencies = [{ name = "anyio" },@@ -798,6 +798,7 @@ dev = [{ name = "pytest" },{ name = "pytest-asyncio" },{ name = "pytest-mock" },+ { name = "pytest-xdist" },{ name = "ruff" },]langchain = [@@ -866,6 +867,7 @@ requires-dist = [{ name = "pytest", marker = "extra == 'dev'", specifier = ">=6.2.5" },{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.26.0" },{ name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.12.0" },+ { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" },{ name = "regex", specifier = ">=2023.10.3" },{ name = "requests", specifier = ">=2.31.0" },{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.3.0" },@@ -900,6 +902,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },]+[[package]]+name = "execnet"+version = "2.1.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },+]+[[package]]name = "fastapi"version = "0.115.13"@@ -2805,6 +2816,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/b2/05/77b60e520511c53d1c1ca75f1930c7dd8e971d0c4379b7f4b3f9644685ba/pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0", size = 9923, upload-time = "2025-05-26T13:58:43.487Z" },]+[[package]]+name = "pytest-xdist"+version = "3.8.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "execnet" },+ { name = "pytest" },+]+sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },+]+[[package]]name = "python-dateutil"version = "2.9.0.post0"