Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

tests: pytest-changed (#6761)

Myles Scolnick07dff1a7045b
diff --git a/.github/workflows/test_be.yaml b/.github/workflows/test_be.yaml
index 7ea6115430..676a491320 100644
--- a/.github/workflows/test_be.yaml
+++ b/.github/workflows/test_be.yaml
@@ -94,6 +94,8 @@ jobs:
uses: styfle/[email protected]
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0 # Fetch all history for git diff
- name: 🥚 Install Hatch
uses: pypa/hatch@install
@@ -119,23 +121,33 @@ jobs:
python-version: ${{ matrix.python-version }}
# Test with base dependencies
- - name: Test with base dependencies
+ - name: Test changed with base dependencies
if: ${{ matrix.dependencies == 'core' }}
run: |
+ # On PRs, compare against base branch; on main, compare against HEAD~1
+ CHANGED_FROM="${{ github.base_ref && format('origin/{0}', github.base_ref) || 'HEAD~1' }}"
hatch run +py=${{ matrix.python-version }} test:test tests/ \
-v \
-k "not test_cli" \
--durations=10 \
+ -p packages.pytest_changed --changed-from=$CHANGED_FROM \
--picked=first
# Test with optional dependencies
- - name: Test with optional dependencies
+ - name: Test changed with optional dependencies
if: ${{ matrix.dependencies == 'core,optional' }}
+ # Include all tests on 3.13 or main
run: |
+ INCLUDE_UNCHANGED=${{ matrix.python-version == '3.13' }} || ${{ github.ref == 'refs/heads/main' }}
+ # On PRs, compare against base branch; on main, compare against HEAD~1
+ CHANGED_FROM="${{ github.base_ref && format('origin/{0}', github.base_ref) || 'HEAD~1' }}"
+
hatch run +py=${{ matrix.python-version }} test-optional:test tests/ \
-v \
-k "not test_cli" \
--durations=10 \
+ -p packages.pytest_changed --changed-from=$CHANGED_FROM \
+ --include-unchanged=$INCLUDE_UNCHANGED \
--picked=first
# Test with minimal dependencies using lowest resolution
@@ -144,11 +156,14 @@ jobs:
- name: Test with minimal dependencies (lowest resolution)
if: ${{ matrix.dependencies == 'minimal' }}
run: |
+ # On PRs, compare against base branch; on main, compare against HEAD~1
+ CHANGED_FROM="${{ github.base_ref && format('origin/{0}', github.base_ref) || 'HEAD~1' }}"
hatch run +py=${{ matrix.python-version }} test:test tests/ \
-v \
-k "not test_cli" \
--durations=10 \
- --picked=first
+ --picked=first \
+ -p packages.pytest_changed --changed-from=$CHANGED_FROM
env:
UV_RESOLUTION: lowest
@@ -177,8 +192,13 @@ jobs:
- name: Test with optional dependencies
run: |
- hatch run +py=3.12 test-optional:test -v tests/ -k "not test_cli" --durations=10 \
- --cov=marimo --cov-report=xml --junitxml=junit.xml -o junit_family=legacy
+ hatch run +py=3.12 test-optional:test \
+ -v \
+ tests/ \
+ -k "not test_cli" \
+ --durations=10 \
+ --picked=first \
+ --cov=marimo --cov-report=xml --junitxml=junit.xml -o junit_family=legacy
# Only upload coverage on `main` so it is not blocking
# and only on `3.12`, `ubuntu-latest`, with optional deps since it is mostly duplicate
diff --git a/marimo/_data/models.py b/marimo/_data/models.py
index 6f5ab9b946..a19931cf5b 100644
--- a/marimo/_data/models.py
+++ b/marimo/_data/models.py
@@ -39,7 +39,7 @@ class DataTableColumn(BaseStruct):
sample_values: list[Any]
def __post_init__(self) -> None:
- # Sometimes like pandas, sqlalchemy or ibis may return column names as objects
+ # Sometimes libraries (like pandas, sqlalchemy or ibis) may return column names as objects
# instead of strings, although their type hints are str
# Instead of trying to track this down each time, just convert to string
self.name = str(self.name)
diff --git a/packages/pytest_changed/README.md b/packages/pytest_changed/README.md
new file mode 100644
index 0000000000..4a073ecbd8
--- /dev/null
+++ b/packages/pytest_changed/README.md
@@ -0,0 +1,70 @@
+# pytest-changed Plugin
+
+A pytest plugin that uses `ruff analyze graph` to intelligently discover and run only the tests affected by code changes.
+
+## Overview
+
+This plugin analyzes your git changes and uses Ruff's dependency graph analysis to determine which tests need to be run. It performs a search through the dependency graph to find all files affected by your changes, then runs only the tests that could be impacted.
+
+## Usage
+
+**Run only tests affected by changes from main branch:**
+
+```bash
+pytest -p packages.pytest_changed --changed-from=main
+```
+
+**Using Hatch:**
+
+```bash
+hatch run +py=3.12 test:test -p packages.pytest_changed --changed-from=main tests/
+```
+
+### Compare Against Different References
+
+```bash
+# Compare against HEAD (staged changes)
+pytest -p packages.pytest_changed --changed-from=HEAD
+
+# Compare against origin/main
+pytest -p packages.pytest_changed --changed-from=origin/main
+```
+
+### Preview What Would Be Run
+
+Use `--collect-only` to see which tests would be selected without running them:
+
+```bash
+pytest -p packages.pytest_changed --changed-from=main --collect-only tests/
+```
+
+### Include Unchanged Tests
+
+Run affected tests plus all other tests:
+
+```bash
+pytest -p packages.pytest_changed --changed-from=main --include-unchanged tests/
+```
+
+### Run Specific Test Directories
+
+The plugin respects pytest's normal path filtering:
+
+```bash
+pytest -p packages.pytest_changed --changed-from=main tests/_runtime/
+```
+
+## How It Works
+
+1. **Find Changes**: Uses `git diff --name-only <ref>` to find Python files that have changed
+2. **Build Graph**: Runs `ruff analyze graph --direction dependents` to build a dependency graph
+3. **Graph Traversal**: Performs BFS from changed files to find all affected files
+4. **Filter Tests**: Identifies which affected files are test files
+5. **Run Tests**: Only runs pytest on the affected test files
+
+## Configuration Options
+
+### Command Line Options
+
+- `--changed-from=<ref>`: Git reference to compare against (required to activate plugin)
+- `--include-unchanged`: Also run tests that haven't changed (optional)
diff --git a/packages/pytest_changed/__init__.py b/packages/pytest_changed/__init__.py
new file mode 100644
index 0000000000..3255d05807
--- /dev/null
+++ b/packages/pytest_changed/__init__.py
@@ -0,0 +1,301 @@
+"""
+Pytest plugin that uses `ruff analyze graph` to discover and run tests
+affected by changes from a git reference.
+
+Usage:
+ pytest --changed-from=HEAD
+ pytest --changed-from=main
+ pytest --changed-from=origin/main
+"""
+
+import json
+import subprocess
+from pathlib import Path
+
+import pytest
+
+print_ = print
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+ """Add command-line options for the plugin."""
+ group = parser.getgroup(
+ "ruff-graph", "Run tests based on dependency graph"
+ )
+ group.addoption(
+ "--changed-from",
+ action="store",
+ dest="changed_from",
+ default=None,
+ help="Git reference to compare against (e.g., HEAD, main, origin/main)",
+ )
+ group.addoption(
+ "--include-unchanged",
+ action="store",
+ dest="include_unchanged",
+ default=False,
+ type=lambda x: x.lower() in ("true", "1", "yes"),
+ help="Also run tests that haven't changed (in addition to affected tests)",
+ )
+
+
+def get_changed_files(ref: str, repo_root: Path) -> set[Path]:
+ """
+ Get list of changed Python files from git diff.
+
+ Args:
+ ref: Git reference to compare against
+ repo_root: Root of the git repository
+
+ Returns:
+ Set of absolute paths to changed Python files
+ """
+ try:
+ result = subprocess.run(
+ ["git", "diff", "--name-only", ref],
+ cwd=repo_root,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+
+ changed_files: set[Path] = set()
+ for line in result.stdout.strip().split("\n"):
+ if not line:
+ continue
+ file_path = repo_root / line
+ if file_path.suffix == ".py" and file_path.exists():
+ changed_files.add(file_path)
+
+ return changed_files
+ except subprocess.CalledProcessError as e:
+ error_msg = f"Failed to get git diff from '{ref}': {e}"
+ if e.stderr:
+ error_msg += f"\nStderr: {e.stderr}"
+ error_msg += (
+ "\n\nTip: Make sure the reference exists. "
+ "In CI, you may need to fetch history with 'fetch-depth: 0' "
+ "in actions/checkout."
+ )
+ pytest.exit(error_msg, returncode=1)
+
+
+def get_dependency_graph(
+ repo_root: Path, direction: str = "dependents"
+) -> dict[str, list[str]]:
+ """
+ Get dependency graph from ruff analyze graph.
+
+ Args:
+ repo_root: Root of the repository
+ direction: Either "dependencies" or "dependents"
+
+ Returns:
+ Dictionary mapping file paths to list of related file paths
+ """
+ try:
+ result = subprocess.run(
+ [
+ "uvx",
+ # uv notes `analyze graph` is experimental,
+ # so we fix the ruff version for now
+ "analyze",
+ "graph",
+ "--detect-string-imports",
+ "--direction",
+ direction,
+ ".",
+ ],
+ cwd=repo_root,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+
+ # Parse JSON output
+ graph: dict[str, list[str]] = json.loads(result.stdout)
+
+ # Convert to absolute paths
+ absolute_graph = {}
+ for file_path, dependencies in graph.items():
+ abs_path = str(repo_root / file_path)
+ abs_deps = [str(repo_root / dep) for dep in dependencies]
+ absolute_graph[abs_path] = abs_deps
+
+ return absolute_graph
+ except subprocess.CalledProcessError as e:
+ pytest.exit(f"Failed to run ruff analyze graph: {e}", returncode=1)
+ except json.JSONDecodeError as e:
+ pytest.exit(f"Failed to parse ruff output: {e}", returncode=1)
+
+
+def find_affected_files(
+ changed_files: set[Path],
+ dependency_graph: dict[str, list[str]],
+) -> set[Path]:
+ """
+ Find all files affected by changes using BFS graph traversal.
+ Handles cycles and deduplication.
+
+ Args:
+ changed_files: Set of files that have changed
+ dependency_graph: Map of file -> files that depend on it
+
+ Returns:
+ Set of all affected file paths (including changed files)
+ """
+ affected: set[Path] = set(changed_files)
+ visited: set[str] = set()
+ queue: list[Path] = list(changed_files)
+
+ while queue:
+ current = queue.pop(0)
+ current_str = str(current)
+
+ if current_str in visited:
+ continue
+ visited.add(current_str)
+
+ # Find all files that depend on this file
+ dependents = dependency_graph.get(current_str, [])
+ for dependent in dependents:
+ dependent_path = Path(dependent)
+ if dependent_path not in affected:
+ affected.add(dependent_path)
+ queue.append(dependent_path)
+
+ return affected
+
+
+def find_test_files(affected_files: set[Path], repo_root: Path) -> set[Path]:
+ """
+ Filter affected files to only include test files.
+
+ Args:
+ affected_files: Set of all affected files
+ repo_root: Root of the repository
+
+ Returns:
+ Set of test file paths
+ """
+ del repo_root
+ test_files: set[Path] = set()
+
+ for file_path in affected_files:
+ # Check if it's a test file
+ if file_path.exists() and (
+ file_path.name.startswith("test_")
+ or file_path.name.endswith("_test.py")
+ or "tests" in file_path.parts
+ ):
+ test_files.add(file_path)
+
+ return test_files
+
+
+def pytest_configure(config: pytest.Config) -> None:
+ """Configure pytest with our plugin."""
+ changed_from: str | None = config.getoption("changed_from")
+
+ if changed_from is None:
+ return
+
+ # Get the repository root
+ try:
+ result = subprocess.run(
+ ["git", "rev-parse", "--show-toplevel"],
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ repo_root = Path(result.stdout.strip())
+ except subprocess.CalledProcessError:
+ pytest.exit("Not in a git repository", returncode=1)
+
+ # Step 1: Find changed files
+ changed_files = get_changed_files(changed_from, repo_root)
+
+ if not changed_files:
+ print_(f"No Python files changed from {changed_from}")
+ config.option.changed_test_files = set()
+ return
+
+ print_(
+ f"Found {len(changed_files)} changed Python files from {changed_from}"
+ )
+ for f in sorted(changed_files):
+ print_(f" - {f.relative_to(repo_root)}")
+
+ # Step 2: Get dependency graph (dependents direction)
+ print_("\nBuilding dependency graph...")
+ dependency_graph = get_dependency_graph(repo_root, direction="dependents")
+
+ # Step 3: Find all affected files (BFS with cycle detection)
+ print_("Finding affected files...")
+ affected_files = find_affected_files(changed_files, dependency_graph)
+ print_(f"Found {len(affected_files)} affected files")
+
+ # Step 4: Filter to test files
+ test_files = find_test_files(affected_files, repo_root)
+
+ if not test_files:
+ print_(f"\nNo tests affected by changes from {changed_from}")
+ config.option.changed_test_files = set()
+ return
+
+ print_(f"\nFound {len(test_files)} affected test files:")
+ for test_file in sorted(test_files):
+ print_(f" - {test_file.relative_to(repo_root)}")
+
+ # Store the test files in config for collection hook
+ config.option.changed_test_files = test_files
+
+
+def pytest_collection_modifyitems(
+ config: pytest.Config, items: list[pytest.Item]
+) -> None:
+ """Modify test collection to only include affected tests."""
+ changed_from: str | None = config.getoption("changed_from")
+
+ if changed_from is None:
+ return
+
+ changed_test_files: set[Path] = getattr(
+ config.option, "changed_test_files", set()
+ )
+
+ if not changed_test_files:
+ # No affected tests, skip all
+ for item in items:
+ item.add_marker(
+ pytest.mark.skip(
+ reason=f"Not affected by changes from {changed_from}"
+ )
+ )
+ return
+
+ include_unchanged = config.getoption("include_unchanged")
+
+ # Filter items to only those in affected test files
+ selected: list[pytest.Item] = []
+ deselected: list[pytest.Item] = []
+
+ for item in items:
+ test_file = Path(item.fspath)
+
+ if test_file in changed_test_files:
+ selected.append(item)
+ elif include_unchanged:
+ # Run all tests if include_unchanged is set
+ selected.append(item)
+ else:
+ deselected.append(item)
+
+ if deselected:
+ config.hook.pytest_deselected(items=deselected)
+ items[:] = selected
+
+ print_(f"\nRunning {len(selected)} tests from affected files")
+ if deselected:
+ print_(f"Skipped {len(deselected)} tests from unaffected files")
diff --git a/tests/_server/api/endpoints/test_execution.py b/tests/_server/api/endpoints/test_execution.py
index 210787a3f5..6e7d19e0e4 100644
--- a/tests/_server/api/endpoints/test_execution.py
+++ b/tests/_server/api/endpoints/test_execution.py
@@ -186,6 +186,7 @@ class TestExecutionRoutes_EditMode:
sys.platform == "win32",
reason="Skipping test on Windows due to websocket issues",
)
+ @pytest.mark.flaky(reruns=5)
@with_session(SESSION_ID)
def test_app_meta_request(client: TestClient) -> None:
response = client.post(
@@ -356,6 +357,7 @@ class TestExecutionRoutes_RunMode:
assert response.status_code == 401, response.text
@staticmethod
+ @pytest.mark.flaky(reruns=5)
@with_session(SESSION_ID)
def test_app_meta_request(client: TestClient) -> None:
response = client.post(