AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).
chore(ci): speed up the test loop (#381)
Rufus584fef46754d
diff --git a/.claude/settings.json b/.claude/settings.jsonindex 5f5ead60..ae204a72 100644--- a/.claude/settings.json+++ b/.claude/settings.json@@ -4,6 +4,7 @@"mcp__github__pull_request_read","Bash(make lint)","Bash(make test)",+ "Bash(make test-changed)","Bash(make check)","Bash(python3 scripts/pr_checks.py:*)","Bash(uv run pytest:*)",diff --git a/.claude/skills/steward/SKILL.md b/.claude/skills/steward/SKILL.mdindex 384a14f9..1196210c 100644--- a/.claude/skills/steward/SKILL.md+++ b/.claude/skills/steward/SKILL.md@@ -8,7 +8,12 @@ description: Repo rules for driving an Open Executive pull request to green - CIOnly what is specific to this repo. `CLAUDE.md` covers the code rules.## Before every push-- Run `make check`. It runs what CI runs (lint, unit and integration tests,+- First push: run `make check` once. Later pushes in the same round: run+ `make test-changed` (the tests covering your change) and `ruff` on the+ files you touched; CI runs the rest. Rerun the full `make check` only if+ the fix touched shared code (`tests/conftest.py`, `pyproject.toml`, a base+ class). CI cancels a superseded run, so don't push piecemeal.+- `make check` It runs what CI runs (lint, unit and integration tests,the UI build if `packages/ui` changed, `scripts/pr_checks.py`) with the`BACKEND_SHARED_SECRET` / `OE_PUBLIC_DEPLOYMENT` traps handled.- If a test fails only in the full run, rerun it serially (no `-n`). Thediff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 09e6b574..ce977e45 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -9,9 +9,51 @@ on:permissions:contents: read+# A new push to a PR supersedes the run in flight: agents often push several+# times in a row, and each superseded run would otherwise burn a full suite.+# Never cancel main, so every merge keeps its own result.+concurrency:+ group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}+jobs:+ # Decides which suites a PR can affect. Skipped jobs count as passing for+ # required checks. Pushes to main run everything.+ changes:+ name: Detect changed areas+ runs-on: ubuntu-latest+ outputs:+ python: ${{ steps.filter.outputs.python }}+ ui: ${{ steps.filter.outputs.ui }}+ steps:+ - uses: actions/checkout@v7+ with:+ fetch-depth: 0+ persist-credentials: false++ - name: Filter by changed paths+ id: filter+ env:+ EVENT: ${{ github.event_name }}+ BASE_REF: ${{ github.base_ref }}+ run: |+ python=true; ui=true+ if [ "$EVENT" = pull_request ]; then+ files=$(git diff --name-only "origin/$BASE_REF"...HEAD)+ # Python suites only skip when every change is inside the web app.+ if ! echo "$files" | grep -qv '^packages/ui/'; then python=false; fi+ # The web app's checks only run when it changed, or when something+ # they read changed: this workflow, and the Python files its+ # parity tests parse (see CLAUDE.md -> UI lint).+ if ! echo "$files" | grep -qE '^(packages/ui/|\.github/workflows/ci\.yml$|packages/core/openexecutive/api/main\.py$|packages/core/openexecutive/utils/deployment\.py$)'; then ui=false; fi+ fi+ echo "python=$python" >> "$GITHUB_OUTPUT"+ echo "ui=$ui" >> "$GITHUB_OUTPUT"+python-lint:name: Python lint & type check+ needs: changes+ if: needs.changes.outputs.python == 'true'runs-on: ubuntu-lateststeps:- uses: actions/checkout@v7@@ -22,7 +64,10 @@ jobs:python-version: "3.11"- name: Install uv- run: pip install uv+ uses: astral-sh/setup-uv@v7+ with:+ enable-cache: true+ cache-dependency-glob: packages/core/uv.lock# docker/Dockerfile installs from uv.lock with `uv sync --locked`, which# fails the image build if the lock is stale. `uv sync` below would@@ -65,6 +110,8 @@ jobs:python-test:name: Python tests+ needs: changes+ if: needs.changes.outputs.python == 'true'runs-on: ubuntu-lateststeps:- uses: actions/checkout@v7@@ -75,7 +122,10 @@ jobs:python-version: "3.11"- name: Install uv- run: pip install uv+ uses: astral-sh/setup-uv@v7+ with:+ enable-cache: true+ cache-dependency-glob: packages/core/uv.lock- name: Install dependenciesrun: cd packages/core && uv sync --extra dev@@ -101,6 +151,8 @@ jobs:ui-typecheck:name: UI type check+ needs: changes+ if: needs.changes.outputs.ui == 'true'runs-on: ubuntu-lateststeps:- uses: actions/checkout@v7@@ -111,6 +163,8 @@ jobs:uses: actions/setup-node@v7with:node-version: "22"+ cache: npm+ cache-dependency-path: packages/ui/package-lock.json- name: Install dependenciesrun: cd packages/ui && npm cidiff --git a/CLAUDE.md b/CLAUDE.mdindex 1d483586..99343df5 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -21,6 +21,7 @@ docker/ Dockerfile + docker-compose.ymlmake dev # Start FastAPI (port 8000) + Next.js (port 3000)make test # Run pytestmake lint # ruff check + mypy+make test-changed # fast loop: only the tests that cover what this branch changedmake check # everything CI checks: lint, unit tests, UI build (if touched), PR rulesmake eval # Run eval suite against localhostmake docker # docker compose up --build@@ -275,6 +276,12 @@ make eval # runs packages/core/openexecutive/evals/_scenarios/*.yaml, writes e## Definition of Done+While iterating, run `make test-changed` (the tests that cover what the branch+changed) plus `ruff` on the files you touched, not the full suite. Run+`make check` **once**, before the first push, then batch fixes: later pushes+rely on CI and rerun only what failed, unless the fix touched shared code+(`tests/conftest.py`, `pyproject.toml`, a base class).+Before calling a code change done or opening a PR:- `make check` passes. It unsets `BACKEND_SHARED_SECRET` / `OE_PUBLIC_DEPLOYMENT` for the tests, so the Testing gotchas above don't bite.diff --git a/Makefile b/Makefileindex d8c3d08c..a61a5f06 100644--- a/Makefile+++ b/Makefile@@ -1,4 +1,4 @@-.PHONY: dev stop test lint check eval docker clean install discord+.PHONY: dev stop test test-changed lint check eval docker clean install discordinstall:cd packages/core && uv sync@@ -49,6 +49,16 @@ stop:test:cd packages/core && uv run pytest tests/ -v --tb=short+# The fast inner loop: only the test files that cover what this branch changed+# (scripts/changed_tests.py), with the same env handling as `check`. Run+# `make check` once before the first push; CI covers the rest.+test-changed:+ @files=$$(python3 scripts/changed_tests.py --base $(BASE)); \+ if [ -z "$$files" ]; then echo "No changed code under packages/core maps to a test file."; exit 0; fi; \+ echo "$$files"; \+ cd packages/core && env -u BACKEND_SHARED_SECRET -u OE_PUBLIC_DEPLOYMENT -u SAAS_MODE \+ uv run pytest $$files -n auto --dist loadfile -q+lint:cd packages/core && uv run ruff check openexecutive/ && uv run mypy openexecutive/diff --git a/packages/core/tests/unit/test_changed_tests.py b/packages/core/tests/unit/test_changed_tests.pynew file mode 100644index 00000000..70405ea7--- /dev/null+++ b/packages/core/tests/unit/test_changed_tests.py@@ -0,0 +1,83 @@+"""Tests for scripts/changed_tests.py, which picks the tests to run for a branch."""++from __future__ import annotations++import importlib.util+import sys+from pathlib import Path++import pytest++_SCRIPT = Path(__file__).resolve().parents[4] / "scripts" / "changed_tests.py"+_spec = importlib.util.spec_from_file_location("changed_tests", _SCRIPT)+assert _spec and _spec.loader+changed_tests = importlib.util.module_from_spec(_spec)+sys.modules["changed_tests"] = changed_tests+_spec.loader.exec_module(changed_tests)++PKG = "packages/core/openexecutive/"+TESTS = "packages/core/tests/"+++def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ """A tiny repo layout: two unit tests and one integration test."""+ core = tmp_path / "packages" / "core"+ for rel, text in {+ "tests/unit/test_episodic.py": "from openexecutive.memory.episodic import x\n",+ "tests/unit/test_slack.py": "import openexecutive.integrations.slack\n",+ "tests/integration/test_chat.py": "from openexecutive.memory import episodic\n"+ "# openexecutive.memory\n",+ }.items():+ path = core / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(text)+ monkeypatch.setattr(changed_tests, "ROOT", tmp_path)+ monkeypatch.setattr(changed_tests, "CORE", core)+ return tmp_path+++def test_dotted_module_path() -> None:+ assert changed_tests.dotted(PKG + "memory/episodic.py") == "openexecutive.memory.episodic"+++def test_dotted_package_init_is_the_package() -> None:+ assert changed_tests.dotted(PKG + "memory/__init__.py") == "openexecutive.memory"+++def test_source_change_selects_tests_that_mention_the_module(repo: Path) -> None:+ picked = changed_tests.select([PKG + "memory/episodic.py"])+ assert picked == ["tests/unit/test_episodic.py"]+++def test_package_init_change_selects_every_test_under_that_package(repo: Path) -> None:+ picked = changed_tests.select([PKG + "memory/__init__.py"])+ assert picked == [+ "tests/integration/test_chat.py",+ "tests/unit/test_episodic.py",+ ]+++def test_changed_test_file_is_selected_itself(repo: Path) -> None:+ picked = changed_tests.select([TESTS + "unit/test_slack.py"])+ assert picked == ["tests/unit/test_slack.py"]+++def test_deleted_test_file_is_not_selected(repo: Path) -> None:+ assert changed_tests.select([TESTS + "unit/test_gone.py"]) == []+++def test_unrelated_change_selects_nothing(repo: Path) -> None:+ assert changed_tests.select(["README.md", "packages/ui/src/app/page.tsx"]) == []+++ "shared",+ [+ "packages/core/tests/conftest.py",+ "packages/core/pyproject.toml",+ "packages/core/uv.lock",+ ],+)+def test_shared_config_change_selects_the_whole_suite(repo: Path, shared: str) -> None:+ assert changed_tests.select([shared]) == ["tests/unit", "tests/integration"]diff --git a/scripts/changed_tests.py b/scripts/changed_tests.pynew file mode 100644index 00000000..700dc45b--- /dev/null+++ b/scripts/changed_tests.py@@ -0,0 +1,79 @@+"""Print the test files that cover what changed on this branch.++ python3 scripts/changed_tests.py [--base origin/main]++The fast inner loop: edit, run these, repeat. `make check` still runs the whole+suite once before the first push. A test file is picked when it is itself+changed, or when its source mentions the dotted module path of a changed+package file (`openexecutive.memory.episodic`), including the package that+re-exports it. A change to shared test or build config selects everything.+Prints nothing when no source file under packages/core changed.+"""++from __future__ import annotations++import argparse+import subprocess+import sys+from pathlib import Path++ROOT = Path(__file__).resolve().parent.parent+CORE = ROOT / "packages" / "core"+TEST_DIRS = ("tests/unit", "tests/integration")+# These affect every test, so a change to one runs the whole suite.+RUN_ALL = ("packages/core/tests/conftest.py", "packages/core/pyproject.toml", "packages/core/uv.lock")+++def git(*args: str) -> list[str]:+ out = subprocess.run(+ ["git", *args], cwd=ROOT, capture_output=True, text=True, check=True+ ).stdout+ return [line for line in out.splitlines() if line]+++def changed_files(base: str) -> list[str]:+ merge_base = git("merge-base", base, "HEAD")[0]+ files = set(git("diff", "--name-only", merge_base))+ files.update(git("ls-files", "--others", "--exclude-standard"))+ return sorted(files)+++def dotted(path: str) -> str:+ """packages/core/openexecutive/a/b.py -> openexecutive.a.b (a/__init__.py -> openexecutive.a)."""+ parts = Path(path).relative_to("packages/core").with_suffix("").parts+ if parts[-1] == "__init__":+ parts = parts[:-1]+ return ".".join(parts)+++def select(files: list[str]) -> list[str]:+ if any(f in RUN_ALL for f in files):+ return [d for d in TEST_DIRS]+ picked: set[str] = set()+ modules: list[str] = []+ for f in files:+ if f.startswith("packages/core/tests/") and f.endswith(".py") and Path(ROOT / f).exists():+ if Path(f).name.startswith("test_"):+ picked.add(str(Path(f).relative_to("packages/core")))+ elif f.startswith("packages/core/openexecutive/") and f.endswith(".py"):+ modules.append(dotted(f))+ if modules:+ for d in TEST_DIRS:+ for test in sorted((CORE / d).glob("test_*.py")):+ text = test.read_text(encoding="utf-8", errors="ignore")+ if any(m in text for m in modules):+ picked.add(str(test.relative_to(CORE)))+ return sorted(picked)+++def main() -> int:+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--base", default="origin/main")+ args = parser.parse_args()+ for path in select(changed_files(args.base)):+ print(path)+ return 0+++if __name__ == "__main__":+ sys.exit(main())