Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

ci(tests): parallelize pytest with xdist and simplify CI jobs (#10078)

Isaac Miller6abd518a2b48
diff --git a/.github/workflows/dependency-range.yml b/.github/workflows/dependency-range.yml
index 8cac7606f..3b15a6119 100644
--- a/.github/workflows/dependency-range.yml
+++ b/.github/workflows/dependency-range.yml
@@ -13,6 +13,10 @@ on:
permissions: {}
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
jobs:
dependency-range:
name: Dependencies (${{ matrix.profile }})
@@ -84,7 +88,7 @@ jobs:
PROFILE: ${{ matrix.profile }}
run: |
set +e
- uv run --frozen --no-sync pytest -q tests/ --extra --deno 2>&1 \
+ uv run --frozen --no-sync pytest -q -n auto --dist worksteal tests/ --extra --deno 2>&1 \
| tee "$RUNNER_TEMP/pytest-$PROFILE.log"
TEST_STATUS=${PIPESTATUS[0]}
set -e
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
index 00898b716..49396ffb2 100644
--- a/.github/workflows/run_tests.yml
+++ b/.github/workflows/run_tests.yml
@@ -9,6 +9,10 @@ on:
permissions: {}
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
jobs:
workflow-lint:
name: Lint GitHub Actions Workflows
@@ -99,16 +103,12 @@ jobs:
run: |
uv sync --dev -p .venv --extra dev
uv pip list
- - name: Run lint with tests
- uses: chartboost/ruff-action@e18ae971ccee1b2d7bbef113930f00c670b78da4 # v1
- with:
- args: check --fix-only
- name: Run tests with pytest
- run: uv run -p .venv pytest -vv tests/
+ run: uv run -p .venv pytest -vv -n auto --dist worksteal tests/
- name: Install optional dependencies
run: uv sync -p .venv --extra dev --extra test_extras
- name: Run extra and deno tests
- run: uv run -p .venv pytest tests/ -m 'extra or deno' --extra --deno
+ run: uv run -p .venv pytest tests/ -m 'extra or deno' --extra --deno -n auto --dist worksteal
llm_call_test:
name: Run Tests with Real LM
@@ -201,15 +201,11 @@ jobs:
cache-dependency-glob: |
**/pyproject.toml
**/uv.lock
- - name: Create and activate virtual environment
- run: |
- uv venv .venv
- echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH"
- - name: Install dependencies
- run: uv sync --dev -p .venv --extra dev
- name: Build
- run: uv run -p .venv python -m build
+ run: uv build
- name: Install built package
- run: uv pip install dist/*.whl -p .venv
+ run: |
+ uv venv .venv
+ uv pip install dist/*.whl -p .venv
- name: Test import dspy
- run: uv run -p .venv python -c "import dspy"
+ run: .venv/bin/python -c "import dspy"
diff --git a/pyproject.toml b/pyproject.toml
index 18240fd40..39353ed93 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,6 +49,7 @@ dev = [
"pytest>=6.2.5",
"pytest-mock>=3.12.0",
"pytest-asyncio>=0.26.0",
+ "pytest-xdist>=3.5.0",
"ruff>=0.3.0",
"pre-commit>=3.7.0",
"pillow>=10.1.0",
diff --git a/tests/test_utils/server/__init__.py b/tests/test_utils/server/__init__.py
index 75535a1d5..dbc735e9e 100644
--- a/tests/test_utils/server/__init__.py
+++ b/tests/test_utils/server/__init__.py
@@ -76,7 +76,7 @@ def _get_random_port():
return s.getsockname()[1]
-def _wait_for_port(host, port, timeout=10):
+def _wait_for_port(host, port, timeout=60):
start_time = time.time()
while time.time() - start_time < timeout:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
diff --git a/uv.lock b/uv.lock
index ac1eb4042..07d5de5d3 100644
--- a/uv.lock
+++ b/uv.lock
@@ -761,7 +761,7 @@ wheels = [
[[package]]
name = "dspy"
-version = "3.2.1"
+version = "3.3.0b1"
source = { editable = "." }
dependencies = [
{ name = "anyio" },
@@ -798,6 +798,7 @@ dev = [
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-mock" },
+ { name = "pytest-xdist" },
{ name = "ruff" },
]
langchain = [
@@ -866,6 +867,7 @@ requires-dist = [
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=6.2.5" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.26.0" },
{ name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.12.0" },
+ { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.5.0" },
{ name = "regex", specifier = ">=2023.10.3" },
{ name = "requests", specifier = ">=2.31.0" },
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.3.0" },
@@ -900,6 +902,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
]
+[[package]]
+name = "execnet"
+version = "2.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" },
+]
+
[[package]]
name = "fastapi"
version = "0.115.13"
@@ -2805,6 +2816,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b2/05/77b60e520511c53d1c1ca75f1930c7dd8e971d0c4379b7f4b3f9644685ba/pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0", size = 9923, upload-time = "2025-05-26T13:58:43.487Z" },
]
+[[package]]
+name = "pytest-xdist"
+version = "3.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "execnet" },
+ { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
+]
+
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"