Skip to content

πŸš€ The fast, Pythonic way to build MCP servers and clients.

Reduce redundant CI execution while preserving required checks (#5049)

nate nowackdf1dc79c5ff7
diff --git a/.github/scripts/test-ci-workflows.mjs b/.github/scripts/test-ci-workflows.mjs
new file mode 100644
index 000000000..c0a7e5e37
--- /dev/null
+++ b/.github/scripts/test-ci-workflows.mjs
@@ -0,0 +1,250 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+
+const repoRoot = fileURLToPath(new URL("../../", import.meta.url));
+const workflows = JSON.parse(
+ execFileSync(
+ "uv",
+ [
+ "run",
+ "--no-sync",
+ "python",
+ "-c",
+ `
+import json
+from pathlib import Path
+import yaml
+# BaseLoader preserves the GitHub Actions 'on' key rather than YAML 1.1's boolean.
+print(json.dumps({name: yaml.load(Path(f'.github/workflows/{name}.yml').read_text(), Loader=yaml.BaseLoader)
+ for name in ['run-tests', 'run-static', 'run-upgrade-checks']}))
+`,
+ ],
+ { cwd: repoRoot, encoding: "utf8" },
+ ),
+);
+const workflow = workflows["run-tests"];
+const script = workflow.jobs.changes.steps[0].with.script;
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+
+async function classify(
+ files,
+ {
+ event = "pull_request",
+ count = files.length,
+ error = false,
+ moved = false,
+ } = {},
+) {
+ const warnings = [];
+ let calls = 0;
+ const result = await new AsyncFunction("github", "context", "core", script)(
+ {
+ rest: {
+ pulls: {
+ listFiles: "listFiles",
+ get: async () => ({
+ data: {
+ head: { sha: moved ? "new-head" : "head" },
+ base: { sha: "base" },
+ },
+ }),
+ },
+ },
+ paginate: async (route, args) => {
+ calls++;
+ assert.equal(route, "listFiles");
+ assert.deepEqual(args, {
+ owner: "PrefectHQ",
+ repo: "fastmcp",
+ pull_number: 42,
+ per_page: 100,
+ });
+ if (error) throw new Error("API unavailable");
+ return files;
+ },
+ },
+ {
+ eventName: event,
+ repo: { owner: "PrefectHQ", repo: "fastmcp" },
+ payload: {
+ pull_request: {
+ number: 42,
+ changed_files: count,
+ head: { sha: "head" },
+ base: { sha: "base" },
+ },
+ },
+ },
+ { warning: (message) => warnings.push(message) },
+ );
+ return { result, calls, warnings };
+}
+
+for (const filename of [
+ "README.md",
+ "CONTRIBUTING.md",
+ "CODE_OF_CONDUCT.md",
+ "docs/clients/tools.mdx",
+ "docs/v3/guide.md",
+]) {
+ test(`editorial change: ${filename}`, async () => {
+ assert.equal((await classify([{ filename }])).result, false);
+ });
+}
+for (const filename of [
+ "fastmcp_slim/fastmcp/server.py",
+ "fastmcp_remote/cli.py",
+ "fastmcp_tasks/extension.py",
+ "tests/test_client.py",
+ "examples/demo.py",
+ ".github/actions/setup-uv/action.yml",
+ ".github/workflows/run-tests.yml",
+ ".github/scripts/test-ci-workflows.mjs",
+ "pyproject.toml",
+ "uv.lock",
+ ".pre-commit-config.yaml",
+ "docs/script.js",
+ "docs/snippets/example.py",
+ "new-package/README.md",
+ "CLAUDE.md",
+]) {
+ test(`full coverage with mixed changes: ${filename}`, async () => {
+ assert.equal(
+ (await classify([{ filename: "README.md" }, { filename }])).result,
+ true,
+ );
+ });
+}
+test("renaming code to documentation still runs tests", async () => {
+ assert.equal(
+ (
+ await classify([
+ { filename: "docs/old.md", previous_filename: "fastmcp_slim/old.py" },
+ ])
+ ).result,
+ true,
+ );
+});
+test("renaming documentation to code still runs tests", async () => {
+ assert.equal(
+ (await classify([{ filename: "new.py", previous_filename: "docs/old.md" }]))
+ .result,
+ true,
+ );
+});
+test("editorial rename can skip tests", async () => {
+ assert.equal(
+ (
+ await classify([
+ { filename: "docs/new.md", previous_filename: "docs/old.md" },
+ ])
+ ).result,
+ false,
+ );
+});
+test("complete paginated editorial diff can skip tests", async () => {
+ assert.equal(
+ (
+ await classify(
+ Array.from({ length: 150 }, (_, i) => ({ filename: `docs/${i}.md` })),
+ )
+ ).result,
+ false,
+ );
+});
+test("empty, incomplete, changed and capped lists run tests", async () => {
+ for (const count of [0, 2, 3001]) {
+ assert.equal(
+ (await classify([{ filename: "README.md" }], { count })).result,
+ true,
+ );
+ }
+ assert.equal((await classify([])).result, true);
+ assert.equal(
+ (
+ await classify(
+ Array.from({ length: 3000 }, () => ({ filename: "README.md" })),
+ )
+ ).result,
+ true,
+ );
+});
+test("API failure falls back to full coverage", async () => {
+ const result = await classify([], { error: true });
+ assert.equal(result.result, true);
+ assert.equal(result.warnings.length, 1);
+});
+for (const event of ["push", "workflow_dispatch"]) {
+ test(`${event} always runs tests without consulting PR files`, async () => {
+ const result = await classify([], { event });
+ assert.equal(result.result, true);
+ assert.equal(result.calls, 0);
+ });
+}
+test("required matrix checks retain names even when editorial steps skip", () => {
+ const matrix = workflow.jobs.run_tests;
+ assert.equal(
+ matrix.name,
+ "Tests: Python ${{ matrix.python-version }} on ${{ matrix.os }}",
+ );
+ assert.deepEqual(matrix.strategy.matrix, {
+ os: ["ubuntu-latest", "windows-latest"],
+ "python-version": ["3.10"],
+ include: [{ os: "ubuntu-latest", "python-version": "3.13" }],
+ });
+ assert.equal(matrix.if, "${{ !cancelled() }}");
+ assert.equal(matrix.steps[0].uses, "actions/checkout@v7");
+ for (const step of matrix.steps.slice(2, 4))
+ assert.equal(step.if, "needs.changes.outputs.run-tests != 'false'");
+ assert.equal(
+ matrix.steps[1].if,
+ "needs.changes.outputs.run-tests != 'false' || (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10')",
+ );
+ assert.equal(
+ matrix.steps[4].if,
+ "needs.changes.outputs.run-tests == 'false' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'",
+ );
+ assert.equal(matrix.steps[4].run, "uv run pytest tests/docs -n 0");
+ for (const name of [
+ "run_tests_lowest_direct",
+ "run_conformance_tests",
+ "run_integration_tests",
+ "package_install_smoke",
+ ]) {
+ assert.equal(workflow.jobs[name].needs, "changes");
+ assert.equal(
+ workflow.jobs[name].if,
+ "${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}",
+ );
+ }
+});
+test("PR cancellation cannot supersede main or manual runs", () => {
+ for (const name of ["run-tests", "run-static"]) {
+ assert.deepEqual(workflows[name].concurrency, {
+ group:
+ "${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}",
+ "cancel-in-progress": "${{ github.event_name == 'pull_request' }}",
+ });
+ assert.deepEqual(workflows[name].on.push, { branches: ["main"] });
+ assert.ok(Object.hasOwn(workflows[name].on, "pull_request"));
+ assert.ok(Object.hasOwn(workflows[name].on, "workflow_dispatch"));
+ }
+ assert.equal(workflows["run-static"].jobs.static_analysis.if, undefined);
+});
+test("upgrade coverage remains nightly and manually dispatchable", () => {
+ assert.deepEqual(workflows["run-upgrade-checks"].on, {
+ schedule: [{ cron: "0 2 * * *" }],
+ workflow_dispatch: "",
+ });
+ assert.ok(workflows["run-upgrade-checks"].jobs.notify);
+ assert.ok(workflows["run-upgrade-checks"].jobs["close-on-success"]);
+});
+
+test("a PR updated during classification gets full coverage", async () => {
+ assert.equal(
+ (await classify([{ filename: "README.md" }], { moved: true })).result,
+ true,
+ );
+});
diff --git a/.github/workflows/run-static.yml b/.github/workflows/run-static.yml
index 19332c11c..40d34ddfa 100644
--- a/.github/workflows/run-static.yml
+++ b/.github/workflows/run-static.yml
@@ -6,21 +6,18 @@ env:
on:
push:
branches: ["main"]
- paths:
- - "fastmcp_slim/**"
- - "fastmcp_remote/**"
- - "tests/**"
- - "examples/**"
- - "pyproject.toml"
- - "uv.lock"
- - ".github/workflows/**"
- - ".github/scripts/test-issue-link.mjs"
# run on all pull requests because these checks are required and will block merges otherwise
pull_request:
workflow_dispatch:
+# Only newer revisions of the same PR supersede a run. Main and manual runs
+# use unique groups so they always complete.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
permissions:
contents: read
@@ -37,8 +34,8 @@ jobs:
with:
resolution: locked
- - name: Test issue-link workflow
- run: node --test .github/scripts/test-issue-link.mjs
+ - name: Test workflows
+ run: node --test .github/scripts/test-*.mjs
- name: Run prek
uses: j178/[email protected]
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index b94e062c7..d573e4550 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -6,24 +6,68 @@ env:
on:
push:
branches: ["main"]
- paths:
- - "fastmcp_slim/**"
- - "fastmcp_remote/**"
- - "tests/**"
- - "pyproject.toml"
- - "uv.lock"
- - ".github/workflows/**"
# run on all pull requests because these checks are required and will block merges otherwise
pull_request:
workflow_dispatch:
+# Only newer revisions of the same PR supersede a run. Main and manual runs
+# use unique groups so they always complete.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
permissions:
contents: read
+ pull-requests: read
jobs:
+ changes:
+ name: Classify test changes
+ runs-on: ubuntu-latest
+ timeout-minutes: 2
+ outputs:
+ run-tests: ${{ steps.classify.outputs.result }}
+ steps:
+ - uses: actions/github-script@v8
+ id: classify
+ with:
+ script: |
+ // Main and manual runs always exercise the full matrix.
+ if (context.eventName !== 'pull_request') return true;
+ const pr = context.payload.pull_request;
+ try {
+ const files = await github.paginate(github.rest.pulls.listFiles, {
+ ...context.repo,
+ pull_number: pr.number,
+ per_page: 100,
+ });
+ const { data: current } = await github.rest.pulls.get({
+ ...context.repo,
+ pull_number: pr.number,
+ });
+ if (current.head.sha !== pr.head.sha || current.base.sha !== pr.base.sha) return true;
+ // The API caps results at 3,000 files. Never skip an incomplete
+ // or empty diff, including when the PR changed during pagination.
+ if (!files.length || files.length !== pr.changed_files || files.length >= 3000) return true;
+ const editorial = (path) =>
+ /^(README|CONTRIBUTING|CODE_OF_CONDUCT)\.md$/.test(path) ||
+ /^docs\/.*\.(md|mdx)$/.test(path);
+ return files.some((file) =>
+ !editorial(file.filename) ||
+ (file.previous_filename && !editorial(file.previous_filename))
+ );
+ } catch (error) {
+ core.warning(`Could not classify changes; running all tests: ${error.message}`);
+ return true;
+ }
+
run_tests:
+ needs: changes
+ # Expand the matrix even for editorial PRs so every required check keeps
+ # its exact name. Skip the expensive steps, not the matrix job.
+ if: ${{ !cancelled() }}
name: "Tests: Python ${{ matrix.python-version }} on ${{ matrix.os }}"
runs-on: ${{ matrix.os }}
strategy:
@@ -37,23 +81,34 @@ jobs:
timeout-minutes: 10
steps:
+ # Keep local composite action definitions available to the runner.
- uses: actions/checkout@v7
- name: Setup uv
+ if: needs.changes.outputs.run-tests != 'false' || (matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10')
uses: ./.github/actions/setup-uv
with:
python-version: ${{ matrix.python-version }}
resolution: locked
- name: Run unit tests
+ if: needs.changes.outputs.run-tests != 'false'
uses: ./.github/actions/run-pytest
- name: Run serial subprocess tests
+ if: needs.changes.outputs.run-tests != 'false'
uses: ./.github/actions/run-pytest
with:
test-type: client_process
+ - name: Run documentation tests
+ if: needs.changes.outputs.run-tests == 'false' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'
+ run: uv run pytest tests/docs -n 0
+
run_tests_lowest_direct:
+ needs: changes
+ # Run even if classification fails, preserving coverage and check names.
+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}
name: "Tests with lowest-direct dependencies"
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -75,6 +130,9 @@ jobs:
test-type: client_process
run_conformance_tests:
+ needs: changes
+ # Run even if classification fails, preserving coverage and check names.
+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}
name: "MCP conformance tests"
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -98,6 +156,9 @@ jobs:
test-type: conformance
run_integration_tests:
+ needs: changes
+ # Run even if classification fails, preserving coverage and check names.
+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}
name: "Integration tests"
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -120,6 +181,9 @@ jobs:
FASTMCP_TEST_AUTH_GITHUB_CLIENT_SECRET: ${{ secrets.FASTMCP_TEST_AUTH_GITHUB_CLIENT_SECRET }}
package_install_smoke:
+ needs: changes
+ # Run even if classification fails, preserving coverage and check names.
+ if: ${{ !cancelled() && needs.changes.outputs.run-tests != 'false' }}
name: "Package install smoke"
runs-on: ubuntu-latest
timeout-minutes: 10
diff --git a/.github/workflows/run-upgrade-checks.yml b/.github/workflows/run-upgrade-checks.yml
index fb532f3ba..f7decf9f1 100644
--- a/.github/workflows/run-upgrade-checks.yml
+++ b/.github/workflows/run-upgrade-checks.yml
@@ -4,15 +4,6 @@ env:
PY_COLORS: 1
on:
- push:
- branches: ["main"]
- paths:
- - "fastmcp_slim/**"
- - "tests/**"
- - "pyproject.toml"
- - "uv.lock"
- - ".github/workflows/**"
-
schedule:
# Run daily at 2 AM UTC
- cron: "0 2 * * *"