π The fast, Pythonic way to build MCP servers and clients.
Reduce redundant CI execution while preserving required checks (#5049)
nate nowackdf1dc79c5ff7
diff --git a/.github/scripts/test-ci-workflows.mjs b/.github/scripts/test-ci-workflows.mjsnew file mode 100644index 000000000..c0a7e5e37--- /dev/null+++ b/.github/scripts/test-ci-workflows.mjs@@ -0,0 +1,250 @@+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import { fileURLToPath } from "node:url";+import test from "node:test";++const repoRoot = fileURLToPath(new URL("../../", import.meta.url));+const workflows = JSON.parse(+ execFileSync(+ "uv",+ [+ "run",+ "--no-sync",+ "python",+ "-c",+ `+import json+from pathlib import Path+import yaml+# BaseLoader preserves the GitHub Actions 'on' key rather than YAML 1.1's boolean.+print(json.dumps({name: yaml.load(Path(f'.github/workflows/{name}.yml').read_text(), Loader=yaml.BaseLoader)+ for name in ['run-tests', 'run-static', 'run-upgrade-checks']}))+`,+ ],+ { cwd: repoRoot, encoding: "utf8" },+ ),+);+const workflow = workflows["run-tests"];+const script = workflow.jobs.changes.steps[0].with.script;+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;++async function classify(+ files,+ {+ event = "pull_request",+ count = files.length,+ error = false,+ moved = false,+ } = {},+) {+ const warnings = [];+ let calls = 0;+ const result = await new AsyncFunction("github", "context", "core", script)(+ {+ rest: {+ pulls: {+ listFiles: "listFiles",+ get: async () => ({+ data: {+ head: { sha: moved ? "new-head" : "head" },+ base: { sha: "base" },+ },+ }),+ },+ },+ paginate: async (route, args) => {+ calls++;+ assert.equal(route, "listFiles");+ assert.deepEqual(args, {+ owner: "PrefectHQ",+ repo: "fastmcp",+ pull_number: 42,+ per_page: 100,+ });+ if (error) throw new Error("API unavailable");+ return files;+ },+ },+ {+ eventName: event,+ repo: { owner: "PrefectHQ", repo: "fastmcp" },+ payload: {+ pull_request: {+ number: 42,+ changed_files: count,+ head: { sha: "head" },+ base: { sha: "base" },+ },+ },+ },+ { warning: (message) => warnings.push(message) },+ );+ return { result, calls, warnings };+}++for (const filename of [+ "README.md",+ "CONTRIBUTING.md",+ "CODE_OF_CONDUCT.md",+ "docs/clients/tools.mdx",+ "docs/v3/guide.md",+]) {+ test(`editorial change: ${filename}`, async () => {+ assert.equal((await classify([{ filename }])).result, false);+ });+}+for (const filename of [+ "fastmcp_slim/fastmcp/server.py",+ "fastmcp_remote/cli.py",+ "fastmcp_tasks/extension.py",+ "tests/test_client.py",+ "examples/demo.py",+ ".github/actions/setup-uv/action.yml",+ ".github/workflows/run-tests.yml",+ ".github/scripts/test-ci-workflows.mjs",+ "pyproject.toml",+ "uv.lock",+ ".pre-commit-config.yaml",+ "docs/script.js",+ "docs/snippets/example.py",+ "new-package/README.md",+ "CLAUDE.md",+]) {+ test(`full coverage with mixed changes: ${filename}`, async () => {+ assert.equal(+ (await classify([{ filename: "README.md" }, { filename }])).result,+ true,+ );+ });+}+test("renaming code to documentation still runs tests", async () => {+ assert.equal(+ (+ await classify([+ { filename: "docs/old.md", previous_filename: "fastmcp_slim/old.py" },+ ])+ ).result,+ true,+ );+});+test("renaming documentation to code still runs tests", async () => {+ assert.equal(+ (await classify([{ filename: "new.py", previous_filename: "docs/old.md" }]))+ .result,+ true,+ );+});+test("editorial rename can skip tests", async () => {+ assert.equal(+ (+ await classify([+ { filename: "docs/new.md", previous_filename: "docs/old.md" },+ ])+ ).result,+ false,+ );+});+test("complete paginated editorial diff can skip tests", async () => {+ assert.equal(+ (+ await classify(+ Array.from({ length: 150 }, (_, i) => ({ filename: `docs/${i}.md` })),+ )+ ).result,+ false,+ );+});+test("empty, incomplete, changed and capped lists run tests", async () => {+ for (const count of [0, 2, 3001]) {+ assert.equal(+ (await classify([{ filename: "README.md" }], { count })).result,+ true,+ );+ }+ assert.equal((await classify([])).result, true);+ assert.equal(+ (+ await classify(+ Array.from({ length: 3000 }, () => ({ filename: "README.md" })),+ )+ ).result,+ true,+ );+});+test("API failure falls back to full coverage", async () => {+ const result = await classify([], { error: true });+ assert.equal(result.result, true);+ assert.equal(result.warnings.length, 1);+});+for (const event of ["push", "workflow_dispatch"]) {+ test(`${event} always runs tests without consulting PR files`, async () => {+ const result = await classify([], { event });+ assert.equal(result.result, true);+ assert.equal(result.calls, 0);+ });+}+test("required matrix checks retain names even when editorial steps skip", () => {+ const matrix = workflow.jobs.run_tests;+ assert.equal(+ matrix.name,+ "Tests: Python ${{ matrix.python-version }} on ${{ matrix.os }}",+ );+ assert.deepEqual(matrix.strategy.matrix, {+ os: ["ubuntu-latest", "windows-latest"],+ "python-version": ["3.10"],+ include: [{ os: "ubuntu-latest", "python-version": "3.13" }],+ });+ assert.equal(matrix.if, "${{ !cancelled() }}");+ assert.equal(matrix.steps[0].uses, "actions/checkout@v7");+ for (const step of matrix.steps.slice(2, 4))+ assert.equal(step.if, "needs.changes.outputs.run-tests != 'false'");+ assert.equal(+ matrix.steps[1].if,+ "needs.changes.outputs.run-tests != 'false' || (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10')",+ );+ assert.equal(+ matrix.steps[4].if,+ "needs.changes.outputs.run-tests == 'false' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'",+ );+ assert.equal(matrix.steps[4].run, "uv run pytest tests/docs -n 0");+ for (const name of [+ "run_tests_lowest_direct",+ "run_conformance_tests",+ "run_integration_tests",+ "package_install_smoke",+ ]) {+ assert.equal(workflow.jobs[name].needs, "changes");+ assert.equal(+ workflow.jobs[name].if,+ "${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}",+ );+ }+});+test("PR cancellation cannot supersede main or manual runs", () => {+ for (const name of ["run-tests", "run-static"]) {+ assert.deepEqual(workflows[name].concurrency, {+ group:+ "${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}",+ "cancel-in-progress": "${{ github.event_name == 'pull_request' }}",+ });+ assert.deepEqual(workflows[name].on.push, { branches: ["main"] });+ assert.ok(Object.hasOwn(workflows[name].on, "pull_request"));+ assert.ok(Object.hasOwn(workflows[name].on, "workflow_dispatch"));+ }+ assert.equal(workflows["run-static"].jobs.static_analysis.if, undefined);+});+test("upgrade coverage remains nightly and manually dispatchable", () => {+ assert.deepEqual(workflows["run-upgrade-checks"].on, {+ schedule: [{ cron: "0 2 * * *" }],+ workflow_dispatch: "",+ });+ assert.ok(workflows["run-upgrade-checks"].jobs.notify);+ assert.ok(workflows["run-upgrade-checks"].jobs["close-on-success"]);+});++test("a PR updated during classification gets full coverage", async () => {+ assert.equal(+ (await classify([{ filename: "README.md" }], { moved: true })).result,+ true,+ );+});diff --git a/.github/workflows/run-static.yml b/.github/workflows/run-static.ymlindex 19332c11c..40d34ddfa 100644--- a/.github/workflows/run-static.yml+++ b/.github/workflows/run-static.yml@@ -6,21 +6,18 @@ env:on:push:branches: ["main"]- paths:- - "fastmcp_slim/**"- - "fastmcp_remote/**"- - "tests/**"- - "examples/**"- - "pyproject.toml"- - "uv.lock"- - ".github/workflows/**"- - ".github/scripts/test-issue-link.mjs"# run on all pull requests because these checks are required and will block merges otherwisepull_request:workflow_dispatch:+# Only newer revisions of the same PR supersede a run. Main and manual runs+# use unique groups so they always complete.+concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}+permissions:contents: read@@ -37,8 +34,8 @@ jobs:with:resolution: locked- - name: Test issue-link workflow- run: node --test .github/scripts/test-issue-link.mjs+ - name: Test workflows+ run: node --test .github/scripts/test-*.mjs- name: Run prekuses: j178/[email protected]diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.ymlindex b94e062c7..d573e4550 100644--- a/.github/workflows/run-tests.yml+++ b/.github/workflows/run-tests.yml@@ -6,24 +6,68 @@ env:on:push:branches: ["main"]- paths:- - "fastmcp_slim/**"- - "fastmcp_remote/**"- - "tests/**"- - "pyproject.toml"- - "uv.lock"- - ".github/workflows/**"# run on all pull requests because these checks are required and will block merges otherwisepull_request:workflow_dispatch:+# Only newer revisions of the same PR supersede a run. Main and manual runs+# use unique groups so they always complete.+concurrency:+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}+permissions:contents: read+ pull-requests: readjobs:+ changes:+ name: Classify test changes+ runs-on: ubuntu-latest+ timeout-minutes: 2+ outputs:+ run-tests: ${{ steps.classify.outputs.result }}+ steps:+ - uses: actions/github-script@v8+ id: classify+ with:+ script: |+ // Main and manual runs always exercise the full matrix.+ if (context.eventName !== 'pull_request') return true;+ const pr = context.payload.pull_request;+ try {+ const files = await github.paginate(github.rest.pulls.listFiles, {+ ...context.repo,+ pull_number: pr.number,+ per_page: 100,+ });+ const { data: current } = await github.rest.pulls.get({+ ...context.repo,+ pull_number: pr.number,+ });+ if (current.head.sha !== pr.head.sha || current.base.sha !== pr.base.sha) return true;+ // The API caps results at 3,000 files. Never skip an incomplete+ // or empty diff, including when the PR changed during pagination.+ if (!files.length || files.length !== pr.changed_files || files.length >= 3000) return true;+ const editorial = (path) =>+ /^(README|CONTRIBUTING|CODE_OF_CONDUCT)\.md$/.test(path) ||+ /^docs\/.*\.(md|mdx)$/.test(path);+ return files.some((file) =>+ !editorial(file.filename) ||+ (file.previous_filename && !editorial(file.previous_filename))+ );+ } catch (error) {+ core.warning(`Could not classify changes; running all tests: ${error.message}`);+ return true;+ }+run_tests:+ needs: changes+ # Expand the matrix even for editorial PRs so every required check keeps+ # its exact name. Skip the expensive steps, not the matrix job.+ if: ${{ !cancelled() }}name: "Tests: Python ${{ matrix.python-version }} on ${{ matrix.os }}"runs-on: ${{ matrix.os }}strategy:@@ -37,23 +81,34 @@ jobs:timeout-minutes: 10steps:+ # Keep local composite action definitions available to the runner.- uses: actions/checkout@v7- name: Setup uv+ if: needs.changes.outputs.run-tests != 'false' || (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10')uses: ./.github/actions/setup-uvwith:python-version: ${{ matrix.python-version }}resolution: locked- name: Run unit tests+ if: needs.changes.outputs.run-tests != 'false'uses: ./.github/actions/run-pytest- name: Run serial subprocess tests+ if: needs.changes.outputs.run-tests != 'false'uses: ./.github/actions/run-pytestwith:test-type: client_process+ - name: Run documentation tests+ if: needs.changes.outputs.run-tests == 'false' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'+ run: uv run pytest tests/docs -n 0+run_tests_lowest_direct:+ needs: changes+ # Run even if classification fails, preserving coverage and check names.+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}name: "Tests with lowest-direct dependencies"runs-on: ubuntu-latesttimeout-minutes: 10@@ -75,6 +130,9 @@ jobs:test-type: client_processrun_conformance_tests:+ needs: changes+ # Run even if classification fails, preserving coverage and check names.+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}name: "MCP conformance tests"runs-on: ubuntu-latesttimeout-minutes: 10@@ -98,6 +156,9 @@ jobs:test-type: conformancerun_integration_tests:+ needs: changes+ # Run even if classification fails, preserving coverage and check names.+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}name: "Integration tests"runs-on: ubuntu-latesttimeout-minutes: 10@@ -120,6 +181,9 @@ jobs:FASTMCP_TEST_AUTH_GITHUB_CLIENT_SECRET: ${{ secrets.FASTMCP_TEST_AUTH_GITHUB_CLIENT_SECRET }}package_install_smoke:+ needs: changes+ # Run even if classification fails, preserving coverage and check names.+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}name: "Package install smoke"runs-on: ubuntu-latesttimeout-minutes: 10diff --git a/.github/workflows/run-upgrade-checks.yml b/.github/workflows/run-upgrade-checks.ymlindex fb532f3ba..f7decf9f1 100644--- a/.github/workflows/run-upgrade-checks.yml+++ b/.github/workflows/run-upgrade-checks.yml@@ -4,15 +4,6 @@ env:PY_COLORS: 1on:- push:- branches: ["main"]- paths:- - "fastmcp_slim/**"- - "tests/**"- - "pyproject.toml"- - "uv.lock"- - ".github/workflows/**"-schedule:# Run daily at 2 AM UTC- cron: "0 2 * * *"