Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

ci: shard API unit test files before collection (#42586)

Stephen Zhou7f018fce7b36
diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml
index 05738455011..0500edfd25f 100644
--- a/.github/workflows/api-tests.yml
+++ b/.github/workflows/api-tests.yml
@@ -56,19 +56,21 @@ jobs:
- name: Run Unit Tests
run: |
+ uv run --project api python api/dev/pytest_sharding.py \
+ --shard-index "${{ matrix.shardIndex }}" \
+ --shard-total "${{ matrix.shardTotal }}" \
+ --ignore api/tests/unit_tests/controllers \
+ api/tests/unit_tests \
+ api/providers/vdb/*/tests/unit_tests \
+ api/providers/trace/*/tests/unit_tests > "$RUNNER_TEMP/unit-test-files.txt"
uv run --project api pytest \
-p no:benchmark \
--timeout "${PYTEST_TIMEOUT:-20}" \
-n auto \
- --shard-index "${{ matrix.shardIndex }}" \
- --shard-total "${{ matrix.shardTotal }}" \
-o 'addopts=--cov=./api --cov-branch --import-mode=importlib' \
--cov-report= \
--junitxml=test-results/unit.xml \
- api/tests/unit_tests \
- api/providers/vdb/*/tests/unit_tests \
- api/providers/trace/*/tests/unit_tests \
- --ignore=api/tests/unit_tests/controllers
+ "@$RUNNER_TEMP/unit-test-files.txt"
# Controller tests register Flask routes at import time, so keep them out of xdist.
uv run --project api pytest \
--timeout "${PYTEST_TIMEOUT:-20}" \
diff --git a/api/dev/pytest_sharding.py b/api/dev/pytest_sharding.py
new file mode 100644
index 00000000000..df6ebf6eda0
--- /dev/null
+++ b/api/dev/pytest_sharding.py
@@ -0,0 +1,43 @@
+"""Select pytest files before collection so workers do not import other shards."""
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+
+def select_test_files(
+ roots: Sequence[Path], *, shard_index: int, shard_total: int, ignored: Sequence[Path] = ()
+) -> list[Path]:
+ """Return a deterministic file shard using pytest's default filename patterns."""
+ if shard_total < 1 or not 1 <= shard_index <= shard_total:
+ raise ValueError("shard-index must be between 1 and shard-total")
+ files = sorted(
+ {
+ path
+ for root in roots
+ for path in root.rglob("*.py")
+ if (path.name.startswith("test_") or path.name.endswith("_test.py"))
+ and not any(path.is_relative_to(directory) for directory in ignored)
+ }
+ )
+ return files[shard_index - 1 :: shard_total]
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--shard-index", type=int, required=True)
+ parser.add_argument("--shard-total", type=int, required=True)
+ parser.add_argument("--ignore", type=Path, action="append", default=[])
+ parser.add_argument("roots", type=Path, nargs="+")
+ args = parser.parse_args()
+ files = select_test_files(
+ args.roots, shard_index=args.shard_index, shard_total=args.shard_total, ignored=args.ignore
+ )
+ if not files:
+ parser.error("No test files selected; refusing to fall back to pytest's default discovery")
+ sys.stdout.write("\n".join(path.as_posix() for path in files) + "\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/api/tests/unit_tests/test_pytest_sharding.py b/api/tests/unit_tests/test_pytest_sharding.py
new file mode 100644
index 00000000000..e81d57e07a3
--- /dev/null
+++ b/api/tests/unit_tests/test_pytest_sharding.py
@@ -0,0 +1,51 @@
+"""Keep early CI sharding complete, disjoint, and independent of input order."""
+
+from pathlib import Path
+
+import pytest
+
+from dev.pytest_sharding import main, select_test_files
+
+
+def test_file_shards_cover_each_test_file_once(tmp_path: Path) -> None:
+ unit = tmp_path / "unit"
+ provider = tmp_path / "provider"
+ controllers = unit / "controllers"
+ expected = {
+ unit / "test_a.py",
+ unit / "nested" / "test_b.py",
+ unit / "nested" / "feature_test.py",
+ provider / "test_a.py",
+ provider / "test_c.py",
+ }
+ for path in expected | {controllers / "test_controller.py", unit / "conftest.py", unit / "helper.py"}:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.touch()
+
+ shards = [
+ select_test_files([unit, provider, unit], shard_index=i, shard_total=3, ignored=[controllers])
+ for i in range(1, 4)
+ ]
+ selected = [path for shard in shards for path in shard]
+ assert len(selected) == len(set(selected))
+ assert set(selected) == expected
+ assert max(map(len, shards)) - min(map(len, shards)) <= 1
+ assert shards[1] == select_test_files([provider, unit], shard_index=2, shard_total=3, ignored=[controllers])
+
+
[email protected](("index", "total"), [(0, 3), (4, 3), (1, 0), (1, -1)])
+def test_rejects_invalid_shard(tmp_path: Path, index: int, total: int) -> None:
+ with pytest.raises(ValueError, match="shard-index"):
+ select_test_files([tmp_path], shard_index=index, shard_total=total)
+
+
+def test_empty_shard_fails_instead_of_running_default_discovery(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+ monkeypatch.setattr("sys.argv", ["pytest_sharding.py", "--shard-index", "1", "--shard-total", "3", str(tmp_path)])
+ with pytest.raises(SystemExit) as exc:
+ main()
+ assert exc.value.code == 2
+ captured = capsys.readouterr()
+ assert captured.out == ""
+ assert "No test files selected" in captured.err