Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

chore(ci): speed up the test loop (#381)

Rufus584fef46754d
diff --git a/.claude/settings.json b/.claude/settings.json
index 5f5ead60..ae204a72 100644
--- a/.claude/settings.json
+++ b/.claude/settings.json
@@ -4,6 +4,7 @@
"mcp__github__pull_request_read",
"Bash(make lint)",
"Bash(make test)",
+ "Bash(make test-changed)",
"Bash(make check)",
"Bash(python3 scripts/pr_checks.py:*)",
"Bash(uv run pytest:*)",
diff --git a/.claude/skills/steward/SKILL.md b/.claude/skills/steward/SKILL.md
index 384a14f9..1196210c 100644
--- a/.claude/skills/steward/SKILL.md
+++ b/.claude/skills/steward/SKILL.md
@@ -8,7 +8,12 @@ description: Repo rules for driving an Open Executive pull request to green - CI
Only what is specific to this repo. `CLAUDE.md` covers the code rules.
## Before every push
-- Run `make check`. It runs what CI runs (lint, unit and integration tests,
+- First push: run `make check` once. Later pushes in the same round: run
+ `make test-changed` (the tests covering your change) and `ruff` on the
+ files you touched; CI runs the rest. Rerun the full `make check` only if
+ the fix touched shared code (`tests/conftest.py`, `pyproject.toml`, a base
+ class). CI cancels a superseded run, so don't push piecemeal.
+- `make check` It runs what CI runs (lint, unit and integration tests,
the UI build if `packages/ui` changed, `scripts/pr_checks.py`) with the
`BACKEND_SHARED_SECRET` / `OE_PUBLIC_DEPLOYMENT` traps handled.
- If a test fails only in the full run, rerun it serially (no `-n`). The
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 09e6b574..ce977e45 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,9 +9,51 @@ on:
permissions:
contents: read
+# A new push to a PR supersedes the run in flight: agents often push several
+# times in a row, and each superseded run would otherwise burn a full suite.
+# Never cancel main, so every merge keeps its own result.
+concurrency:
+ group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
jobs:
+ # Decides which suites a PR can affect. Skipped jobs count as passing for
+ # required checks. Pushes to main run everything.
+ changes:
+ name: Detect changed areas
+ runs-on: ubuntu-latest
+ outputs:
+ python: ${{ steps.filter.outputs.python }}
+ ui: ${{ steps.filter.outputs.ui }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Filter by changed paths
+ id: filter
+ env:
+ EVENT: ${{ github.event_name }}
+ BASE_REF: ${{ github.base_ref }}
+ run: |
+ python=true; ui=true
+ if [ "$EVENT" = pull_request ]; then
+ files=$(git diff --name-only "origin/$BASE_REF"...HEAD)
+ # Python suites only skip when every change is inside the web app.
+ if ! echo "$files" | grep -qv '^packages/ui/'; then python=false; fi
+ # The web app's checks only run when it changed, or when something
+ # they read changed: this workflow, and the Python files its
+ # parity tests parse (see CLAUDE.md -> UI lint).
+ if ! echo "$files" | grep -qE '^(packages/ui/|\.github/workflows/ci\.yml$|packages/core/openexecutive/api/main\.py$|packages/core/openexecutive/utils/deployment\.py$)'; then ui=false; fi
+ fi
+ echo "python=$python" >> "$GITHUB_OUTPUT"
+ echo "ui=$ui" >> "$GITHUB_OUTPUT"
+
python-lint:
name: Python lint & type check
+ needs: changes
+ if: needs.changes.outputs.python == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
@@ -22,7 +64,10 @@ jobs:
python-version: "3.11"
- name: Install uv
- run: pip install uv
+ uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+ cache-dependency-glob: packages/core/uv.lock
# docker/Dockerfile installs from uv.lock with `uv sync --locked`, which
# fails the image build if the lock is stale. `uv sync` below would
@@ -65,6 +110,8 @@ jobs:
python-test:
name: Python tests
+ needs: changes
+ if: needs.changes.outputs.python == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
@@ -75,7 +122,10 @@ jobs:
python-version: "3.11"
- name: Install uv
- run: pip install uv
+ uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+ cache-dependency-glob: packages/core/uv.lock
- name: Install dependencies
run: cd packages/core && uv sync --extra dev
@@ -101,6 +151,8 @@ jobs:
ui-typecheck:
name: UI type check
+ needs: changes
+ if: needs.changes.outputs.ui == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
@@ -111,6 +163,8 @@ jobs:
uses: actions/setup-node@v7
with:
node-version: "22"
+ cache: npm
+ cache-dependency-path: packages/ui/package-lock.json
- name: Install dependencies
run: cd packages/ui && npm ci
diff --git a/CLAUDE.md b/CLAUDE.md
index 1d483586..99343df5 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -21,6 +21,7 @@ docker/ Dockerfile + docker-compose.yml
make dev # Start FastAPI (port 8000) + Next.js (port 3000)
make test # Run pytest
make lint # ruff check + mypy
+make test-changed # fast loop: only the tests that cover what this branch changed
make check # everything CI checks: lint, unit tests, UI build (if touched), PR rules
make eval # Run eval suite against localhost
make docker # docker compose up --build
@@ -275,6 +276,12 @@ make eval # runs packages/core/openexecutive/evals/_scenarios/*.yaml, writes e
## Definition of Done
+While iterating, run `make test-changed` (the tests that cover what the branch
+changed) plus `ruff` on the files you touched, not the full suite. Run
+`make check` **once**, before the first push, then batch fixes: later pushes
+rely on CI and rerun only what failed, unless the fix touched shared code
+(`tests/conftest.py`, `pyproject.toml`, a base class).
+
Before calling a code change done or opening a PR:
- `make check` passes. It unsets `BACKEND_SHARED_SECRET` / `OE_PUBLIC_DEPLOYMENT` for the tests, so the Testing gotchas above don't bite.
diff --git a/Makefile b/Makefile
index d8c3d08c..a61a5f06 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: dev stop test lint check eval docker clean install discord
+.PHONY: dev stop test test-changed lint check eval docker clean install discord
install:
cd packages/core && uv sync
@@ -49,6 +49,16 @@ stop:
test:
cd packages/core && uv run pytest tests/ -v --tb=short
+# The fast inner loop: only the test files that cover what this branch changed
+# (scripts/changed_tests.py), with the same env handling as `check`. Run
+# `make check` once before the first push; CI covers the rest.
+test-changed:
+ @files=$$(python3 scripts/changed_tests.py --base $(BASE)); \
+ if [ -z "$$files" ]; then echo "No changed code under packages/core maps to a test file."; exit 0; fi; \
+ echo "$$files"; \
+ cd packages/core && env -u BACKEND_SHARED_SECRET -u OE_PUBLIC_DEPLOYMENT -u SAAS_MODE \
+ uv run pytest $$files -n auto --dist loadfile -q
+
lint:
cd packages/core && uv run ruff check openexecutive/ && uv run mypy openexecutive/
diff --git a/packages/core/tests/unit/test_changed_tests.py b/packages/core/tests/unit/test_changed_tests.py
new file mode 100644
index 00000000..70405ea7
--- /dev/null
+++ b/packages/core/tests/unit/test_changed_tests.py
@@ -0,0 +1,83 @@
+"""Tests for scripts/changed_tests.py, which picks the tests to run for a branch."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+
+_SCRIPT = Path(__file__).resolve().parents[4] / "scripts" / "changed_tests.py"
+_spec = importlib.util.spec_from_file_location("changed_tests", _SCRIPT)
+assert _spec and _spec.loader
+changed_tests = importlib.util.module_from_spec(_spec)
+sys.modules["changed_tests"] = changed_tests
+_spec.loader.exec_module(changed_tests)
+
+PKG = "packages/core/openexecutive/"
+TESTS = "packages/core/tests/"
+
+
+def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ """A tiny repo layout: two unit tests and one integration test."""
+ core = tmp_path / "packages" / "core"
+ for rel, text in {
+ "tests/unit/test_episodic.py": "from openexecutive.memory.episodic import x\n",
+ "tests/unit/test_slack.py": "import openexecutive.integrations.slack\n",
+ "tests/integration/test_chat.py": "from openexecutive.memory import episodic\n"
+ "# openexecutive.memory\n",
+ }.items():
+ path = core / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(text)
+ monkeypatch.setattr(changed_tests, "ROOT", tmp_path)
+ monkeypatch.setattr(changed_tests, "CORE", core)
+ return tmp_path
+
+
+def test_dotted_module_path() -> None:
+ assert changed_tests.dotted(PKG + "memory/episodic.py") == "openexecutive.memory.episodic"
+
+
+def test_dotted_package_init_is_the_package() -> None:
+ assert changed_tests.dotted(PKG + "memory/__init__.py") == "openexecutive.memory"
+
+
+def test_source_change_selects_tests_that_mention_the_module(repo: Path) -> None:
+ picked = changed_tests.select([PKG + "memory/episodic.py"])
+ assert picked == ["tests/unit/test_episodic.py"]
+
+
+def test_package_init_change_selects_every_test_under_that_package(repo: Path) -> None:
+ picked = changed_tests.select([PKG + "memory/__init__.py"])
+ assert picked == [
+ "tests/integration/test_chat.py",
+ "tests/unit/test_episodic.py",
+ ]
+
+
+def test_changed_test_file_is_selected_itself(repo: Path) -> None:
+ picked = changed_tests.select([TESTS + "unit/test_slack.py"])
+ assert picked == ["tests/unit/test_slack.py"]
+
+
+def test_deleted_test_file_is_not_selected(repo: Path) -> None:
+ assert changed_tests.select([TESTS + "unit/test_gone.py"]) == []
+
+
+def test_unrelated_change_selects_nothing(repo: Path) -> None:
+ assert changed_tests.select(["README.md", "packages/ui/src/app/page.tsx"]) == []
+
+
+ "shared",
+ [
+ "packages/core/tests/conftest.py",
+ "packages/core/pyproject.toml",
+ "packages/core/uv.lock",
+ ],
+)
+def test_shared_config_change_selects_the_whole_suite(repo: Path, shared: str) -> None:
+ assert changed_tests.select([shared]) == ["tests/unit", "tests/integration"]
diff --git a/scripts/changed_tests.py b/scripts/changed_tests.py
new file mode 100644
index 00000000..700dc45b
--- /dev/null
+++ b/scripts/changed_tests.py
@@ -0,0 +1,79 @@
+"""Print the test files that cover what changed on this branch.
+
+ python3 scripts/changed_tests.py [--base origin/main]
+
+The fast inner loop: edit, run these, repeat. `make check` still runs the whole
+suite once before the first push. A test file is picked when it is itself
+changed, or when its source mentions the dotted module path of a changed
+package file (`openexecutive.memory.episodic`), including the package that
+re-exports it. A change to shared test or build config selects everything.
+Prints nothing when no source file under packages/core changed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+CORE = ROOT / "packages" / "core"
+TEST_DIRS = ("tests/unit", "tests/integration")
+# These affect every test, so a change to one runs the whole suite.
+RUN_ALL = ("packages/core/tests/conftest.py", "packages/core/pyproject.toml", "packages/core/uv.lock")
+
+
+def git(*args: str) -> list[str]:
+ out = subprocess.run(
+ ["git", *args], cwd=ROOT, capture_output=True, text=True, check=True
+ ).stdout
+ return [line for line in out.splitlines() if line]
+
+
+def changed_files(base: str) -> list[str]:
+ merge_base = git("merge-base", base, "HEAD")[0]
+ files = set(git("diff", "--name-only", merge_base))
+ files.update(git("ls-files", "--others", "--exclude-standard"))
+ return sorted(files)
+
+
+def dotted(path: str) -> str:
+ """packages/core/openexecutive/a/b.py -> openexecutive.a.b (a/__init__.py -> openexecutive.a)."""
+ parts = Path(path).relative_to("packages/core").with_suffix("").parts
+ if parts[-1] == "__init__":
+ parts = parts[:-1]
+ return ".".join(parts)
+
+
+def select(files: list[str]) -> list[str]:
+ if any(f in RUN_ALL for f in files):
+ return [d for d in TEST_DIRS]
+ picked: set[str] = set()
+ modules: list[str] = []
+ for f in files:
+ if f.startswith("packages/core/tests/") and f.endswith(".py") and Path(ROOT / f).exists():
+ if Path(f).name.startswith("test_"):
+ picked.add(str(Path(f).relative_to("packages/core")))
+ elif f.startswith("packages/core/openexecutive/") and f.endswith(".py"):
+ modules.append(dotted(f))
+ if modules:
+ for d in TEST_DIRS:
+ for test in sorted((CORE / d).glob("test_*.py")):
+ text = test.read_text(encoding="utf-8", errors="ignore")
+ if any(m in text for m in modules):
+ picked.add(str(test.relative_to(CORE)))
+ return sorted(picked)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--base", default="origin/main")
+ args = parser.parse_args()
+ for path in select(changed_files(args.base)):
+ print(path)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())