Skip to content

πŸš€ The fast, Pythonic way to build MCP servers and clients.

Run downstream smoke against built wheels and pydantic-ai's own MCP tests (#5209)

nate nowackf9f4ac892dd9
diff --git a/.github/workflows/run-downstream-smoke.yml b/.github/workflows/run-downstream-smoke.yml
index 4fc01a049..8476d9b6b 100644
--- a/.github/workflows/run-downstream-smoke.yml
+++ b/.github/workflows/run-downstream-smoke.yml
@@ -1,16 +1,25 @@
name: Downstream smoke
-# Drive this checkout's FastMCP through the frameworks that depend on it, using
-# their latest releases, so a change that breaks them fails here first.
+# Drive this checkout's FastMCP through the frameworks that depend on it, the
+# way uv's ecosystem tests drive downstream projects: build once, install that
+# build where the consumer runs, and run the consumer's own usage against it.
+#
+# Pushes and PRs use the pinned versions below, so a failure points at the
+# change under test. The nightly run resolves each consumer's latest release
+# to catch breakage that ships downstream.
env:
PY_COLORS: 1
+ PYDANTIC_AI_VERSION: "2.47.0"
+ LANGCHAIN_MCP_ADAPTERS_VERSION: "0.3.2"
+ LANGCHAIN_VERSION: "1.4.2"
on:
push:
branches: ["main"]
paths:
- "fastmcp_slim/**"
+ - "fastmcp_tasks/**"
- "tests/downstream/**"
- "pyproject.toml"
- "uv.lock"
@@ -19,16 +28,21 @@ on:
pull_request:
paths:
- "fastmcp_slim/**"
+ - "fastmcp_tasks/**"
- "tests/downstream/**"
- "pyproject.toml"
- "uv.lock"
- ".github/workflows/run-downstream-smoke.yml"
schedule:
- # Downstream releases can break us without any change here.
- cron: "30 2 * * *"
workflow_dispatch:
+ inputs:
+ latest:
+ description: "Test each consumer's latest release instead of the pins"
+ type: boolean
+ default: false
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
@@ -38,31 +52,162 @@ permissions:
contents: read
jobs:
+ plan:
+ name: Plan
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ outputs:
+ fastmcp: ${{ steps.build.outputs.version }}
+ pydantic-ai: ${{ steps.versions.outputs.pydantic-ai }}
+ langchain-mcp-adapters: ${{ steps.versions.outputs.langchain-mcp-adapters }}
+ langchain: ${{ steps.versions.outputs.langchain }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ fetch-depth: 0
+
+ - name: Install uv
+ uses: astral-sh/[email protected]
+
+ - name: Build wheels
+ id: build
+ run: |
+ for package in . fastmcp_slim fastmcp_tasks; do
+ uv build --quiet --wheel --out-dir dist "$package"
+ done
+ version=$(basename dist/fastmcp_slim-*.whl | cut -d- -f2)
+ echo "version=$version" >> "$GITHUB_OUTPUT"
+ ls dist
+
+ - uses: actions/upload-artifact@v7
+ with:
+ name: fastmcp-wheels
+ path: dist/*.whl
+
+ - name: Choose consumer versions
+ id: versions
+ env:
+ LATEST: ${{ github.event_name == 'schedule' || inputs.latest == true }}
+ run: |
+ pick() {
+ if [ "$LATEST" = true ]; then
+ curl -fsS "https://pypi.org/pypi/$1/json" | jq -r .info.version
+ else
+ echo "$2"
+ fi
+ }
+ {
+ echo "pydantic-ai=$(pick pydantic-ai-slim "$PYDANTIC_AI_VERSION")"
+ echo "langchain-mcp-adapters=$(pick langchain-mcp-adapters "$LANGCHAIN_MCP_ADAPTERS_VERSION")"
+ echo "langchain=$(pick langchain "$LANGCHAIN_VERSION")"
+ } | tee -a "$GITHUB_OUTPUT" > versions.txt
+ {
+ echo "### Downstream smoke plan"
+ echo
+ echo "FastMCP \`$(basename dist/fastmcp_slim-*.whl | cut -d- -f2)\` against $([ "$LATEST" = true ] && echo "each consumer's latest release" || echo "pinned consumer versions")"
+ echo
+ echo "| consumer | version |"
+ echo "|---|---|"
+ sed 's/^\(.*\)=\(.*\)$/| \1 | `\2` |/' versions.txt
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ pydantic-ai-suite:
+ name: pydantic-ai test suite
+ needs: plan
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ path: fastmcp
+
+ - uses: actions/checkout@v7
+ with:
+ repository: pydantic/pydantic-ai
+ ref: v${{ needs.plan.outputs.pydantic-ai }}
+ path: pydantic-ai
+ persist-credentials: false
+
+ - uses: actions/download-artifact@v8
+ with:
+ name: fastmcp-wheels
+ path: dist
+
+ - name: Install uv
+ uses: astral-sh/[email protected]
+ with:
+ python-version: "3.13"
+ enable-cache: true
+ cache-dependency-glob: pydantic-ai/uv.lock
+
+ # Mirrors pydantic-ai's own "test FastMCP 4 compatibility" job: their locked
+ # environment, with FastMCP overlaid from the wheels under test.
+ - name: Resolve environment
+ working-directory: pydantic-ai
+ run: |
+ echo "UV_RUN=uv run --all-extras --no-extra mcp-tasks --all-packages \
+ --with fastmcp@file://$(ls $GITHUB_WORKSPACE/dist/fastmcp-4*.whl) \
+ --with fastmcp-slim@file://$(ls $GITHUB_WORKSPACE/dist/fastmcp_slim-*.whl) \
+ --with fastmcp-tasks@file://$(ls $GITHUB_WORKSPACE/dist/fastmcp_tasks-*.whl)" >> "$GITHUB_ENV"
+
+ - name: Check the build under test is installed
+ working-directory: pydantic-ai
+ run: >-
+ $UV_RUN --quiet python -c "import fastmcp, mcp, importlib.metadata as m;
+ assert fastmcp.__version__ == '${{ needs.plan.outputs.fastmcp }}', fastmcp.__version__;
+ print('fastmcp', fastmcp.__version__, 'Β· mcp', m.version('mcp'))"
+
+ - name: Run pydantic-ai's MCP tests
+ working-directory: pydantic-ai
+ run: >-
+ $UV_RUN --quiet pytest -p no:cacheprovider -q
+ --junitxml=$GITHUB_WORKSPACE/junit.xml
+ tests/test_mcp.py tests/models/test_mcp_sampling.py tests/test_conftest.py
+
+ - name: Summarize
+ if: always() && hashFiles('junit.xml') != ''
+ run: python3 fastmcp/tests/downstream/_harness.py junit.xml "pydantic-ai ${{ needs.plan.outputs.pydantic-ai }} MCP test suite"
+
smoke:
- name: ${{ matrix.name }}
+ name: ${{ matrix.consumer }} smoke
+ needs: plan
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
include:
- # pydantic-ai builds on fastmcp.Client, so it shares FastMCP's environment.
- - name: pydantic-ai
- run: uv run --with 'pydantic-ai-slim[mcp]' tests/downstream/smoke_pydantic_ai.py
- # langchain-mcp-adapters pins mcp<2, so it runs in its own environment and reaches FastMCP over the wire.
- - name: langchain
- run: uv run tests/downstream/smoke_langchain.py
-
+ # Client-only, as `pydantic-ai-slim[mcp]` installs it, with the build under test.
+ - consumer: pydantic-ai
+ script: smoke_pydantic_ai.py
+ with: >-
+ --with "pydantic-ai-slim[mcp]==${{ needs.plan.outputs.pydantic-ai }}"
+ --with "fastmcp-slim[client] @ file://$GITHUB_WORKSPACE/$(ls dist/fastmcp_slim-*.whl)"
+ # The adapters pin mcp<2, so they reach FastMCP over the wire.
+ - consumer: langchain
+ script: smoke_langchain.py
+ with: >-
+ --with "langchain-mcp-adapters==${{ needs.plan.outputs.langchain-mcp-adapters }}"
+ --with "langchain==${{ needs.plan.outputs.langchain }}"
+ env:
+ FASTMCP_EXPECTED_VERSION: ${{ needs.plan.outputs.fastmcp }}
steps:
- uses: actions/checkout@v7
+ # The servers every smoke connects to run from this environment.
- name: Setup uv
uses: ./.github/actions/setup-uv
with:
python-version: "3.12"
- - name: Run ${{ matrix.name }} smoke
- run: ${{ matrix.run }}
- env:
- PYDANTIC_AI_NO_BANNER: 1
- FASTMCP_LOG_LEVEL: WARNING
+ - uses: actions/download-artifact@v8
+ with:
+ name: fastmcp-wheels
+ path: dist
+
+ # Resolved outside the project and without its config, as a user's install is.
+ - name: Resolve ${{ matrix.consumer }} environment
+ run: uv run --isolated --no-project --no-config ${{ matrix.with }} python -c pass
+
+ - name: Run ${{ matrix.consumer }} smoke
+ run: uv run --quiet --isolated --no-project --no-config ${{ matrix.with }} python tests/downstream/${{ matrix.script }}
diff --git a/tests/downstream/README.md b/tests/downstream/README.md
new file mode 100644
index 000000000..4ba87bbcf
--- /dev/null
+++ b/tests/downstream/README.md
@@ -0,0 +1,35 @@
+# downstream smoke
+
+Checks that FastMCP still works the way its consumers use it. The
+[workflow](../../.github/workflows/run-downstream-smoke.yml) builds FastMCP's
+wheels once, installs them where each consumer runs, and drives the consumer
+against servers started from the checkout.
+
+| job | what it runs | FastMCP install |
+|---|---|---|
+| pydantic-ai test suite | pydantic-ai's own `tests/test_mcp.py` and sampling tests at a pinned tag | wheels overlaid on pydantic-ai's locked environment, as their FastMCP 4 job does |
+| pydantic-ai smoke | `smoke_pydantic_ai.py` over stdio, `mcp.json` config, streamable HTTP and SSE | client-only `fastmcp-slim[client]`, as `pydantic-ai-slim[mcp]` installs it |
+| langchain smoke | `smoke_langchain.py` over stdio, streamable HTTP and SSE | none: the adapters pin `mcp<2`, so they reach FastMCP over the wire |
+
+HTTP and SSE servers require a bearer token. Each smoke prints one line per
+check, writes a table to the job summary, and reports warnings raised in the
+consumer's process.
+
+Pushes and PRs test the versions pinned in the workflow. The nightly run, or a
+manual run with `latest`, tests each consumer's latest release.
+
+## running locally
+
+```bash
+uv sync
+uv build --wheel --out-dir dist fastmcp_slim
+
+uv run --isolated --no-project --no-config \
+ --with 'pydantic-ai-slim[mcp]' \
+ --with "fastmcp-slim[client] @ file://$PWD/$(ls dist/fastmcp_slim-*.whl)" \
+ python tests/downstream/smoke_pydantic_ai.py
+
+uv run --isolated --no-project --no-config \
+ --with langchain-mcp-adapters --with langchain \
+ python tests/downstream/smoke_langchain.py
+```
diff --git a/tests/downstream/_harness.py b/tests/downstream/_harness.py
new file mode 100644
index 000000000..19f58e902
--- /dev/null
+++ b/tests/downstream/_harness.py
@@ -0,0 +1,216 @@
+"""Shared plumbing for the downstream smoke scripts: server processes and reporting."""
+
+import os
+import secrets
+import socket
+import subprocess
+import sys
+import time
+import traceback
+import warnings
+from collections.abc import Awaitable, Callable, Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+
+HERE = Path(__file__).parent
+SERVER = HERE / "server.py"
+REPO = HERE.parents[1]
+INSTRUCTIONS = "Arithmetic and weather for smoke tests."
+IN_ACTIONS = os.environ.get("GITHUB_ACTIONS") == "true"
+os.environ.setdefault("PYDANTIC_AI_NO_BANNER", "1")
+
+
+def server_python() -> str:
+ """The interpreter of the FastMCP checkout, which is what serves every smoke test."""
+ return os.environ.get("FASTMCP_SERVER_PYTHON") or str(
+ REPO / ".venv" / "bin" / "python"
+ )
+
+
+def server_env() -> dict[str, str]:
+ """Keep the server's own logs and warnings out of the consumer's output."""
+ return {**os.environ, "FASTMCP_LOG_LEVEL": "CRITICAL", "PYTHONWARNINGS": "ignore"}
+
+
+def stdio_command() -> tuple[str, list[str]]:
+ return server_python(), [str(SERVER), "stdio"]
+
+
+@dataclass
+class Server:
+ url: str
+ token: str
+
+ @property
+ def headers(self) -> dict[str, str]:
+ return {"Authorization": f"Bearer {self.token}"}
+
+
+@contextmanager
+def serve(transport: str) -> Iterator[Server]:
+ """Run server.py over `http` or `sse` with bearer auth for the length of the block."""
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ port = sock.getsockname()[1]
+ token = secrets.token_urlsafe(16)
+ proc = subprocess.Popen(
+ [server_python(), str(SERVER), transport, str(port), token], env=server_env()
+ )
+ try:
+ deadline = time.monotonic() + 30
+ while True:
+ if proc.poll() is not None:
+ raise RuntimeError(f"{transport} server exited with {proc.returncode}")
+ try:
+ socket.create_connection(("127.0.0.1", port), timeout=0.2).close()
+ break
+ except OSError:
+ if time.monotonic() > deadline:
+ raise RuntimeError(
+ f"{transport} server did not listen within 30s"
+ ) from None
+ time.sleep(0.1)
+ path = "/sse" if transport == "sse" else "/mcp"
+ yield Server(url=f"http://127.0.0.1:{port}{path}", token=token)
+ finally:
+ proc.terminate()
+ proc.wait(timeout=10)
+
+
+@dataclass
+class Result:
+ transport: str
+ check: str
+ error: str | None
+ seconds: float
+
+
+@dataclass
+class Report:
+ consumer: str
+ versions: dict[str, str]
+ results: list[Result] = field(default_factory=list)
+ warnings: dict[str, str] = field(default_factory=dict)
+ _transport: str = ""
+
+ @contextmanager
+ def transport(self, name: str) -> Iterator[None]:
+ self._transport = name
+ print(
+ f"::group::{self.consumer} via {name}"
+ if IN_ACTIONS
+ else f"\n{self.consumer} via {name}"
+ )
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ try:
+ yield
+ except Exception as error:
+ self._record("connect", error, 0.0)
+ for w in caught:
+ if issubclass(w.category, (DeprecationWarning, UserWarning)):
+ self.warnings.setdefault(f"{w.category.__name__}: {w.message}", name)
+ if IN_ACTIONS:
+ print("::endgroup::")
+
+ async def check(self, name: str, fn: Callable[[], Awaitable[object]]) -> None:
+ start = time.monotonic()
+ try:
+ await fn()
+ error = None
+ except Exception as exc:
+ error = exc
+ self._record(name, error, time.monotonic() - start)
+
+ def _record(self, name: str, error: BaseException | None, seconds: float) -> None:
+ detail = None if error is None else f"{type(error).__name__}: {error}"
+ self.results.append(Result(self._transport, name, detail, seconds))
+ mark = "βœ“" if error is None else "βœ—"
+ print(
+ f" {mark} {name:<34} {seconds * 1000:6.0f} ms"
+ + ("" if error is None else f"\n {detail}")
+ )
+ if error is not None:
+ traceback.print_exception(error, file=sys.stdout)
+
+ def finish(self) -> None:
+ failed = [r for r in self.results if r.error]
+ transports = list(dict.fromkeys(r.transport for r in self.results))
+ checks = list(dict.fromkeys(r.check for r in self.results))
+ cell = {
+ (r.transport, r.check): ("βœ…" if not r.error else "❌")
+ for r in self.results
+ }
+
+ lines = [
+ f"### {'❌' if failed else 'βœ…'} {self.consumer}",
+ "",
+ " Β· ".join(f"`{k} {v}`" for k, v in self.versions.items()),
+ "",
+ "| check | " + " | ".join(transports) + " |",
+ "|---|" + "---|" * len(transports),
+ *(
+ f"| {c} | "
+ + " | ".join(cell.get((t, c), "β€”") for t in transports)
+ + " |"
+ for c in checks
+ ),
+ ]
+ if failed:
+ lines += ["", "**Failures**", ""]
+ lines += [f"- `{r.transport}` / `{r.check}`: {r.error}" for r in failed]
+ if self.warnings:
+ lines += [
+ "",
+ "<details><summary>Warnings raised in the consumer's process</summary>",
+ "",
+ ]
+ lines += [
+ f"- `{where}`: {message}" for message, where in self.warnings.items()
+ ]
+ lines += ["", "</details>"]
+ summary = "\n".join(lines) + "\n"
+
+ print(f"\n{len(self.results) - len(failed)}/{len(self.results)} checks passed")
+ for message, where in self.warnings.items():
+ print(
+ f"::warning title={self.consumer} via {where}::{message}"
+ if IN_ACTIONS
+ else f"warning via {where}: {message}"
+ )
+ if path := os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(path, "a") as fh:
+ fh.write(summary)
+ if failed:
+ sys.exit(1)
+
+
+def junit_summary(path: str, title: str) -> None:
+ """Append a pass/fail line and any failures from a pytest JUnit report to the job summary."""
+ import xml.etree.ElementTree as ET
+
+ suite = ET.parse(path).getroot()
+ suite = suite if suite.tag == "testsuite" else suite[0]
+ total, failures, errors, skipped = (
+ int(suite.get(k, 0)) for k in ("tests", "failures", "errors", "skipped")
+ )
+ failed = [
+ f"- `{case.get('classname')}::{case.get('name')}`"
+ for case in suite.iter("testcase")
+ if case.find("failure") is not None or case.find("error") is not None
+ ]
+ mark = "❌" if failed else "βœ…"
+ lines = [
+ f"### {mark} {title}",
+ "",
+ f"{total - failures - errors - skipped} passed, {failures + errors} failed, {skipped} skipped",
+ *(["", *failed] if failed else []),
+ ]
+ if summary := os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(summary, "a") as fh:
+ fh.write("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+ junit_summary(sys.argv[1], sys.argv[2])
diff --git a/tests/downstream/_http.py b/tests/downstream/_http.py
deleted file mode 100644
index 70f420db5..000000000
--- a/tests/downstream/_http.py
+++ /dev/null
@@ -1,40 +0,0 @@
-"""Serve server.py over streamable HTTP in a subprocess for the length of a `with` block."""
-
-import os
-import socket
-import subprocess
-import sys
-import time
-from collections.abc import Iterator
-from contextlib import contextmanager
-from pathlib import Path
-
-SERVER = Path(__file__).with_name("server.py")
-
-
-def free_port() -> int:
- with socket.socket() as sock:
- sock.bind(("127.0.0.1", 0))
- return sock.getsockname()[1]
-
-
-@contextmanager
-def http_server(python: list[str]) -> Iterator[str]:
- port = free_port()
- proc = subprocess.Popen([*python, str(SERVER), "http", str(port)], env=os.environ)
- try:
- deadline = time.monotonic() + 30
- while time.monotonic() < deadline:
- if proc.poll() is not None:
- sys.exit(f"server exited with {proc.returncode}")
- try:
- socket.create_connection(("127.0.0.1", port), timeout=0.2).close()
- break
- except OSError:
- time.sleep(0.1)
- else:
- sys.exit("server did not start listening within 30s")
- yield f"http://127.0.0.1:{port}/mcp"
- finally:
- proc.terminate()
- proc.wait(timeout=10)
diff --git a/tests/downstream/server.py b/tests/downstream/server.py
index 0cf931cf8..e28ee8f3d 100644
--- a/tests/downstream/server.py
+++ b/tests/downstream/server.py
@@ -1,117 +1,176 @@
"""The FastMCP server every downstream smoke script connects to.
-Run as `python server.py stdio` or `python server.py http <port>`.
+python server.py stdio
+python server.py http <port> <bearer-token>
+python server.py sse <port> <bearer-token>
"""
+import base64
import sys
from dataclasses import dataclass
+from _harness import INSTRUCTIONS
from mcp.types import (
ElicitRequest,
ElicitRequestFormParams,
ElicitResult,
+ EmbeddedResource,
InputRequiredResult,
+ ResourceLink,
+ TextContent,
+ TextResourceContents,
)
from mcp_types.version import MODERN_PROTOCOL_VERSIONS
from pydantic import BaseModel
from fastmcp import Context, FastMCP
from fastmcp.exceptions import ToolError
+from fastmcp.server.auth.providers.jwt import StaticTokenVerifier
+from fastmcp.utilities.types import Audio, Image
-mcp = FastMCP(
- "downstream-smoke", instructions="Arithmetic and weather for smoke tests."
+PNG = base64.b64decode(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="
+)
+WAV = (
+ b"RIFF&\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00D\xac\x00\x00"
+ b"\x88X\x01\x00\x02\x00\x10\x00data\x02\x00\x00\x00\x00\x00"
)
-class Forecast(BaseModel):
- city: str
- celsius: float
- conditions: list[str]
-
-
-def add(a: int, b: int) -> int:
- """Add two integers."""
- return a + b
-
-
-def forecast(city: str, days: int = 1) -> Forecast:
- """Structured weather forecast for a city."""
- return Forecast(city=city, celsius=21.5, conditions=["sunny"] * days)
-
-
-def divide(a: float, b: float) -> float:
- """Divide a by b."""
- if b == 0:
- raise ToolError("cannot divide by zero")
- return a / b
-
-
-async def count_to(n: int, ctx: Context) -> str:
- """Count to n, reporting progress and logging along the way."""
- for i in range(1, n + 1):
- await ctx.report_progress(i, n)
- await ctx.info(f"counted to {n}")
- return f"counted to {n}"
-
-
-@dataclass
-class Approval:
- approved: bool
-
-
-async def confirm(action: str, ctx: Context) -> str | InputRequiredResult:
- """Ask the client to approve an action, using whichever elicitation path the connection's protocol era supports."""
- rc = ctx.request_context
- if rc is not None and rc.protocol_version in MODERN_PROTOCOL_VERSIONS:
- if ctx.input_responses is None:
- form = ElicitRequestFormParams(
- message=f"approve {action}?",
- requested_schema={
- "type": "object",
- "properties": {"approved": {"type": "boolean"}},
- "required": ["approved"],
- },
- )
- return InputRequiredResult(
- result_type="input_required",
- input_requests={
- "approval": ElicitRequest(method="elicitation/create", params=form)
- },
- )
- answer = ctx.input_responses["approval"]
- assert isinstance(answer, ElicitResult), answer
- if answer.action == "accept" and answer.content and answer.content["approved"]:
+def build(auth_token: str | None = None) -> FastMCP:
+ auth = (
+ StaticTokenVerifier({auth_token: {"client_id": "smoke", "scopes": []}})
+ if auth_token
+ else None
+ )
+ mcp = FastMCP("downstream-smoke", instructions=INSTRUCTIONS, auth=auth)
+
+ class Forecast(BaseModel):
+ city: str
+ celsius: float
+ conditions: list[str]
+
+ @dataclass
+ class Approval:
+ approved: bool
+
+ @mcp.tool
+ def add(a: int, b: int) -> int:
+ """Add two integers."""
+ return a + b
+
+ @mcp.tool
+ def forecast(city: str, days: int = 1) -> Forecast:
+ """Structured weather forecast for a city."""
+ return Forecast(city=city, celsius=21.5, conditions=["sunny"] * days)
+
+ @mcp.tool
+ def divide(a: float, b: float) -> float:
+ """Divide a by b."""
+ if b == 0:
+ raise ToolError("cannot divide by zero")
+ return a / b
+
+ @mcp.tool
+ async def count_to(n: int, ctx: Context) -> str:
+ """Count to n, reporting progress and logging along the way."""
+ for i in range(1, n + 1):
+ await ctx.report_progress(i, n)
+ await ctx.info(f"counted to {n}")
+ return f"counted to {n}"
+
+ @mcp.tool
+ def snapshot() -> Image:
+ """A 1x1 PNG."""
+ return Image(data=PNG, format="png")
+
+ @mcp.tool
+ def chime() -> Audio:
+ """A silent WAV."""
+ return Audio(data=WAV, format="wav")
+
+ @mcp.tool
+ def attachments() -> list:
+ """Mixed content: text, an embedded resource, and a resource link."""
+ return [
+ TextContent(type="text", text="see attached"),
+ EmbeddedResource(
+ type="resource",
+ resource=TextResourceContents(
+ uri="notes://today", mime_type="text/plain", text="buy milk"
+ ),
+ ),
+ ResourceLink(
+ type="resource_link",
+ uri="config://app",
+ name="app config",
+ mime_type="application/json",
+ ),
+ ]
+
+ @mcp.tool
+ async def confirm(action: str, ctx: Context) -> str | InputRequiredResult:
+ """Ask the client to approve an action, using whichever elicitation path the connection's protocol era supports."""
+ rc = ctx.request_context
+ if rc is not None and rc.protocol_version in MODERN_PROTOCOL_VERSIONS:
+ if ctx.input_responses is None:
+ form = ElicitRequestFormParams(
+ message=f"approve {action}?",
+ requested_schema={
+ "type": "object",
+ "properties": {"approved": {"type": "boolean"}},
+ "required": ["approved"],
+ },
+ )
+ return InputRequiredResult(
+ result_type="input_required",
+ input_requests={
+ "approval": ElicitRequest(
+ method="elicitation/create", params=form
+ )
+ },
+ )
+ answer = ctx.input_responses["approval"]
+ assert isinstance(answer, ElicitResult), answer
+ if (
+ answer.action == "accept"
+ and answer.content
+ and answer.content["approved"]
+ ):
+ return f"{action}: approved"
+ return f"{action}: {answer.action}"
+ result = await ctx.elicit(f"approve {action}?", response_type=Approval)
+ if result.action == "accept" and result.data.approved:
return f"{action}: approved"
- return f"{action}: {answer.action}"
- result = await ctx.elicit(f"approve {action}?", response_type=Approval)
- if result.action == "accept" and result.data.approved:
- return f"{action}: approved"
- return f"{action}: {result.action}"
-
+ return f"{action}: {result.action}"
[email protected]("config://app", mime_type="application/json")
-def app_config() -> str:
- return '{"mode": "smoke"}'
+ @mcp.resource("config://app", mime_type="application/json")
+ def app_config() -> str:
+ return '{"mode": "smoke"}'
+ @mcp.resource("greeting://{name}")
+ def greeting(name: str) -> str:
+ return f"hello, {name}"
[email protected]("greeting://{name}")
-def greeting(name: str) -> str:
- return f"hello, {name}"
+ @mcp.prompt
+ def review(code: str) -> str:
+ """Ask for a code review."""
+ return f"please review:\n{code}"
-
-def review(code: str) -> str:
- """Ask for a code review."""
- return f"please review:\n{code}"
+ return mcp
if __name__ == "__main__":
- if sys.argv[1:2] == ["http"]:
- mcp.run("http", host="127.0.0.1", port=int(sys.argv[2]), show_banner=False)
- else:
- mcp.run("stdio", show_banner=False)
+ match sys.argv[1:]:
+ case [] | ["stdio"]:
+ build().run("stdio", show_banner=False)
+ case ["http" | "sse" as transport, port, token]:
+ build(token).run(
+ transport,
+ host="127.0.0.1",
+ port=int(port),
+ show_banner=False,
+ log_level="critical",
+ )
+ case args:
+ sys.exit(f"usage: server.py [stdio | http|sse <port> <token>], got {args}")
diff --git a/tests/downstream/smoke_langchain.py b/tests/downstream/smoke_langchain.py
index d070bac36..d4bb32917 100644
--- a/tests/downstream/smoke_langchain.py
+++ b/tests/downstream/smoke_langchain.py
@@ -1,30 +1,36 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = ["langchain-mcp-adapters", "langchain"]
-# ///
-"""Exercise a FastMCP server through langchain-mcp-adapters.
+"""Drive a FastMCP server through langchain-mcp-adapters, the way LangChain users reach it.
The adapters pin `mcp<2` while FastMCP 4 needs `mcp>=2`, so they cannot share an
-environment. This script runs in its own environment and reaches the server
-over the wire, the way LangChain users do:
+environment. This runs in its own environment against servers started from the
+checkout, which also covers handshake-era (mcp v1) clients:
- uv run tests/downstream/smoke_langchain.py
+ uv run --isolated --no-project --with langchain-mcp-adapters --with langchain \
+ python tests/downstream/smoke_langchain.py
"""
import asyncio
+from importlib.metadata import version
from typing import Any
-from _http import SERVER, http_server
+from _harness import INSTRUCTIONS, Report, serve, server_env, stdio_command
from langchain.agents import create_agent
from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
-from langchain_core.messages import AIMessage
+from langchain_core.messages import AIMessage, ToolMessage
from langchain_mcp_adapters.callbacks import Callbacks
from langchain_mcp_adapters.client import MultiServerMCPClient
from langchain_mcp_adapters.tools import load_mcp_tools
from mcp.types import ElicitResult
-REPO = SERVER.parents[2]
-FASTMCP_PYTHON = ["uv", "run", "--project", str(REPO), "--frozen", "python"]
+TOOLS = {
+ "add",
+ "forecast",
+ "divide",
+ "count_to",
+ "snapshot",
+ "chime",
+ "attachments",
+ "confirm",
+}
class ScriptedToolModel(GenericFakeChatModel):
@@ -53,7 +59,14 @@ def scripted_model() -> ScriptedToolModel:
)
-async def exercise(label: str, connection: dict[str, Any]) -> None:
+async def call(tool: Any, args: dict[str, Any]) -> ToolMessage:
+ """Invoke a tool the way an agent does, so the result is a ToolMessage with its artifact."""
+ return await tool.ainvoke(
+ {"type": "tool_call", "name": tool.name, "args": args, "id": "call-1"}
+ )
+
+
+async def exercise(report: Report, connection: dict[str, Any]) -> None:
progress: list[float] = []
logs: list[str] = []
@@ -78,70 +91,150 @@ async def exercise(label: str, connection: dict[str, Any]) -> None:
on_elicitation=on_elicitation,
),
)
+ tools: dict[str, Any] = {}
- tools = {tool.name: tool for tool in await client.get_tools()}
- assert {"add", "forecast", "divide", "count_to", "confirm"} <= set(tools), tools
+ async def lists_tools() -> None:
+ tools.update({tool.name: tool for tool in await client.get_tools()})
+ assert TOOLS <= set(tools), sorted(tools)
- agent = create_agent(scripted_model(), list(tools.values()))
- state = await agent.ainvoke(
- {"messages": [("user", "add 2 and 3, then get the Chicago forecast")]}
- )
- returned = {m.name: m.content for m in state["messages"] if m.type == "tool"}
- assert "5" in str(returned["add"]), returned
- assert "sunny" in str(returned["forecast"]), returned
+ async def server_info() -> None:
+ info = (await client.get_server_info(server_name="smoke"))["smoke"]
+ assert info.instructions == INSTRUCTIONS, info.instructions
+ assert info.serverInfo.name == "downstream-smoke", info.serverInfo
- assert "42" in str(await tools["add"].ainvoke({"a": 40, "b": 2}))
- failed = await tools["divide"].ainvoke(
- {
- "type": "tool_call",
- "name": "divide",
- "args": {"a": 1, "b": 0},
- "id": "call-divide",
- }
- )
- assert failed.status == "error" and "divide by zero" in str(failed.content), failed
+ async def agent_loop() -> None:
+ agent = create_agent(scripted_model(), list(tools.values()))
+ state = await agent.ainvoke(
+ {"messages": [("user", "add 2 and 3, then get the Chicago forecast")]}
+ )
+ returned = {m.name: m.content for m in state["messages"] if m.type == "tool"}
+ assert "5" in str(returned["add"]) and "sunny" in str(returned["forecast"]), (
+ returned
+ )
- assert "deploy: approved" in str(
- await tools["confirm"].ainvoke({"action": "deploy"})
- )
- assert "counted to 3" in str(await tools["count_to"].ainvoke({"n": 3}))
- assert progress == [1, 2, 3], progress
- assert any("counted to 3" in line for line in logs), logs
+ async def structured_output() -> None:
+ message = await call(tools["forecast"], {"city": "Oslo"})
+ expected = {"city": "Oslo", "celsius": 21.5, "conditions": ["sunny"]}
+ assert message.artifact == {"structured_content": expected}, message.artifact
- async with client.session("smoke") as session:
- assert {"add", "forecast"} <= {
- tool.name for tool in await load_mcp_tools(session)
- }
+ async def tool_error() -> None:
+ message = await call(tools["divide"], {"a": 1, "b": 0})
+ assert message.status == "error" and "divide by zero" in str(message.content), (
+ message
+ )
- blobs = await client.get_resources(
- "smoke", uris=["config://app", "greeting://nate"]
- )
- assert [blob.as_string() for blob in blobs] == [
- '{"mode": "smoke"}',
- "hello, nate",
- ], blobs
+ async def image() -> None:
+ [block] = (await call(tools["snapshot"], {})).content
+ assert (
+ block["type"] == "image"
+ and block["mime_type"] == "image/png"
+ and block["base64"]
+ ), block
+
+ async def mixed_content() -> None:
+ blocks = (await call(tools["attachments"], {})).content
+ shapes = [(b["type"], b.get("text") or b.get("url")) for b in blocks]
+ assert shapes == [
+ ("text", "see attached"),
+ ("text", "buy milk"),
+ ("file", "config://app"),
+ ], shapes
+
+ async def elicitation() -> None:
+ assert "deploy: approved" in str(
+ (await call(tools["confirm"], {"action": "deploy"})).content
+ )
- messages = await client.get_prompt("smoke", "review", arguments={"code": "x = 1"})
- assert "x = 1" in str(messages[0].content), messages
- print(f"ok langchain via {label}")
+ async def progress_and_logging() -> None:
+ assert "counted to 3" in str((await call(tools["count_to"], {"n": 3})).content)
+ assert progress == [1, 2, 3], progress
+ assert any("counted to 3" in line for line in logs), logs
+
+ async def explicit_session() -> None:
+ async with client.session("smoke") as session:
+ assert TOOLS <= {tool.name for tool in await load_mcp_tools(session)}
+
+ async def resources() -> None:
+ [blob] = await client.get_resources("smoke", uris=["config://app"])
+ assert (
+ blob.as_string() == '{"mode": "smoke"}'
+ and blob.mimetype == "application/json"
+ ), blob
+
+ async def resource_template() -> None:
+ [blob] = await client.get_resources("smoke", uris=["greeting://nate"])
+ assert blob.as_string() == "hello, nate", blob
+
+ async def prompt() -> None:
+ messages = await client.get_prompt(
+ "smoke", "review", arguments={"code": "x = 1"}
+ )
+ assert "x = 1" in str(messages[0].content), messages
+
+ for check in (
+ lists_tools,
+ server_info,
+ agent_loop,
+ structured_output,
+ tool_error,
+ image,
+ mixed_content,
+ elicitation,
+ progress_and_logging,
+ explicit_session,
+ resources,
+ resource_template,
+ prompt,
+ ):
+ await report.check(check.__name__.replace("_", " "), check)
async def main() -> None:
- from importlib.metadata import version
-
- print(
- f"langchain-mcp-adapters {version('langchain-mcp-adapters')}, mcp {version('mcp')}"
- )
- await exercise(
- "stdio",
+ report = Report(
+ "langchain",
{
- "transport": "stdio",
- "command": FASTMCP_PYTHON[0],
- "args": [*FASTMCP_PYTHON[1:], str(SERVER), "stdio"],
+ "langchain-mcp-adapters": version("langchain-mcp-adapters"),
+ "langchain": version("langchain"),
+ "mcp": version("mcp"),
},
)
- with http_server(FASTMCP_PYTHON) as url:
- await exercise("streamable HTTP", {"transport": "streamable_http", "url": url})
+ command, args = stdio_command()
+ with report.transport("stdio"):
+ await exercise(
+ report,
+ {
+ "transport": "stdio",
+ "command": command,
+ "args": args,
+ "env": server_env(),
+ },
+ )
+
+ for transport, label, kind in (
+ ("http", "streamable HTTP", "streamable_http"),
+ ("sse", "SSE", "sse"),
+ ):
+ with report.transport(label), serve(transport) as server:
+
+ async def rejects_missing_token() -> None:
+ client = MultiServerMCPClient(
+ {"smoke": {"transport": kind, "url": server.url}}
+ )
+ try:
+ await client.get_tools()
+ except BaseException as error:
+ if isinstance(error, KeyboardInterrupt):
+ raise
+ return
+ raise AssertionError("listed tools without a bearer token")
+
+ await report.check("rejects missing bearer token", rejects_missing_token)
+ await exercise(
+ report,
+ {"transport": kind, "url": server.url, "headers": server.headers},
+ )
+
+ report.finish()
if __name__ == "__main__":
diff --git a/tests/downstream/smoke_pydantic_ai.py b/tests/downstream/smoke_pydantic_ai.py
index a8f7494b5..a0d0d2573 100644
--- a/tests/downstream/smoke_pydantic_ai.py
+++ b/tests/downstream/smoke_pydantic_ai.py
@@ -1,17 +1,26 @@
-"""Exercise FastMCP through pydantic-ai's MCPToolset, which is built on fastmcp.Client.
+"""Drive FastMCP through pydantic-ai's MCPToolset, as a pydantic-ai user installs it.
-Runs inside the FastMCP project environment with pydantic-ai layered on top:
+`pydantic-ai-slim[mcp]` depends on the client-only `fastmcp-slim[client]`, so
+this runs in its own environment with no FastMCP server dependencies, resolved
+together with the build under test. The servers run from the checkout:
- uv run --with 'pydantic-ai-slim[mcp]' tests/downstream/smoke_pydantic_ai.py
+ uv build --wheel --out-dir dist fastmcp_slim
+ uv run --isolated --no-project --with 'pydantic-ai-slim[mcp]' \
+ --with "fastmcp-slim[client] @ file://$PWD/$(ls dist/fastmcp_slim-*.whl)" \
+ python tests/downstream/smoke_pydantic_ai.py
"""
import asyncio
-import sys
+import json
+import os
+import tempfile
+from importlib.metadata import version
+from pathlib import Path
from typing import Any
-from _http import SERVER, http_server
-from pydantic_ai import Agent, ModelRetry
-from pydantic_ai.mcp import MCPToolset
+from _harness import INSTRUCTIONS, Report, serve, server_env, stdio_command
+from pydantic_ai import Agent, BinaryContent, ModelRetry
+from pydantic_ai.mcp import MCPToolset, load_mcp_toolsets
from pydantic_ai.messages import (
ModelMessage,
ModelResponse,
@@ -20,9 +29,20 @@ from pydantic_ai.messages import (
ToolReturnPart,
)
from pydantic_ai.models.function import AgentInfo, FunctionModel
-from server import mcp
import fastmcp
+from fastmcp.client.transports import StdioTransport
+
+TOOLS = {
+ "add",
+ "forecast",
+ "divide",
+ "count_to",
+ "snapshot",
+ "chime",
+ "attachments",
+ "confirm",
+}
def scripted_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse:
@@ -34,10 +54,13 @@ def scripted_model(messages: list[ModelMessage], info: AgentInfo) -> ModelRespon
if isinstance(p, ToolReturnPart)
]
if not returns:
+ # load_mcp_toolsets prefixes tool names with the server's config key.
+ name = {t.name.rsplit("_", 1)[-1]: t.name for t in info.function_tools}
+ name.update({t.name: t.name for t in info.function_tools})
return ModelResponse(
parts=[
- ToolCallPart("add", {"a": 2, "b": 3}),
- ToolCallPart("forecast", {"city": "Chicago", "days": 2}),
+ ToolCallPart(name["add"], {"a": 2, "b": 3}),
+ ToolCallPart(name["forecast"], {"city": "Chicago", "days": 2}),
]
)
return ModelResponse(parts=[TextPart(" | ".join(str(p.content) for p in returns))])
@@ -49,7 +72,9 @@ async def accept(
return {"approved": True}
-async def exercise(label: str, target: Any) -> None:
+async def exercise(
+ report: Report, target: Any, headers: dict[str, str] | None = None
+) -> None:
progress: list[float] = []
logs: list[str] = []
@@ -63,57 +88,166 @@ async def exercise(label: str, target: Any) -> None:
toolset = MCPToolset(
target,
+ headers=headers,
+ include_instructions=True,
elicitation_handler=accept,
progress_handler=on_progress,
log_handler=on_log,
)
async with toolset:
- names = {tool.name for tool in await toolset.list_tools()}
- assert {"add", "forecast", "divide", "count_to", "confirm"} <= names, names
- agent = Agent(FunctionModel(scripted_model), toolsets=[toolset])
- result = await agent.run("add 2 and 3, then get the Chicago forecast")
- assert "5" in result.output and "sunny" in result.output, result.output
+ async def lists_tools() -> None:
+ names = {tool.name for tool in await toolset.list_tools()}
+ assert TOOLS <= names, names
+
+ async def instructions() -> None:
+ assert toolset.instructions == INSTRUCTIONS, toolset.instructions
+
+ async def agent_loop() -> None:
+ agent = Agent(FunctionModel(scripted_model), toolsets=[toolset])
+ result = await agent.run("add 2 and 3, then get the Chicago forecast")
+ assert "5" in result.output and "sunny" in result.output, result.output
+
+ async def structured_output() -> None:
+ forecast = await toolset.direct_call_tool("forecast", {"city": "Oslo"})
+ assert forecast == {
+ "city": "Oslo",
+ "celsius": 21.5,
+ "conditions": ["sunny"],
+ }, forecast
+
+ async def tool_error() -> None:
+ try:
+ await toolset.direct_call_tool("divide", {"a": 1, "b": 0})
+ except ModelRetry as error:
+ assert "divide by zero" in str(error), error
+ else:
+ raise AssertionError("divide by zero did not raise ModelRetry")
+
+ async def image() -> None:
+ result = await toolset.direct_call_tool("snapshot", {})
+ assert (
+ isinstance(result, BinaryContent) and result.media_type == "image/png"
+ ), result
+
+ async def audio() -> None:
+ result = await toolset.direct_call_tool("chime", {})
+ assert (
+ isinstance(result, BinaryContent) and result.media_type == "audio/wav"
+ ), result
+
+ async def mixed_content() -> None:
+ result = await toolset.direct_call_tool("attachments", {})
+ assert result == ["see attached", "buy milk", "config://app"], result
+
+ async def elicitation() -> None:
+ result = await toolset.direct_call_tool("confirm", {"action": "deploy"})
+ assert result == "deploy: approved", result
+
+ async def progress_and_logging() -> None:
+ assert (
+ await toolset.direct_call_tool("count_to", {"n": 3}) == "counted to 3"
+ )
+ assert progress == [1, 2, 3], progress
+ assert any("counted to 3" in line for line in logs), logs
+
+ async def resources() -> None:
+ assert "config://app" in {
+ str(r.uri) for r in await toolset.list_resources()
+ }
+ assert await toolset.read_resource("config://app") == '{"mode": "smoke"}'
+
+ async def resource_template() -> None:
+ templates = {
+ t.uri_template for t in await toolset.list_resource_templates()
+ }
+ assert "greeting://{name}" in templates, templates
+ assert await toolset.read_resource("greeting://nate") == "hello, nate"
+
+ async def prompt() -> None:
+ result = await toolset.get_prompt("review", {"code": "x = 1"})
+ assert "x = 1" in str(result.messages[0].content), result
+
+ for check in (
+ lists_tools,
+ instructions,
+ agent_loop,
+ structured_output,
+ tool_error,
+ image,
+ audio,
+ mixed_content,
+ elicitation,
+ progress_and_logging,
+ resources,
+ resource_template,
+ prompt,
+ ):
+ await report.check(check.__name__.replace("_", " "), check)
- assert await toolset.direct_call_tool("add", {"a": 40, "b": 2}) == 42
- forecast = await toolset.direct_call_tool("forecast", {"city": "Oslo"})
- assert forecast == {"city": "Oslo", "celsius": 21.5, "conditions": ["sunny"]}, (
- forecast
- )
- try:
- await toolset.direct_call_tool("divide", {"a": 1, "b": 0})
- except ModelRetry as error:
- assert "divide by zero" in str(error), error
- else:
- raise AssertionError("divide by zero did not raise")
+async def main() -> None:
+ report = Report(
+ "pydantic-ai",
+ {
+ "fastmcp": fastmcp.__version__,
+ "pydantic-ai-slim": version("pydantic-ai-slim"),
+ "mcp": version("mcp"),
+ },
+ )
- assert (
- await toolset.direct_call_tool("confirm", {"action": "deploy"})
- == "deploy: approved"
+ async def installed_build_is_under_test() -> None:
+ expected = os.environ.get("FASTMCP_EXPECTED_VERSION")
+ assert expected in (None, fastmcp.__version__), (
+ f"expected {expected}, got {fastmcp.__version__}"
)
- assert await toolset.direct_call_tool("count_to", {"n": 3}) == "counted to 3"
- assert progress == [1, 2, 3], progress
- assert any("counted to 3" in line for line in logs), logs
-
- assert "config://app" in {str(r.uri) for r in await toolset.list_resources()}
- assert await toolset.read_resource("config://app") == '{"mode": "smoke"}'
- assert await toolset.read_resource("greeting://nate") == "hello, nate"
-
- prompt = await toolset.get_prompt("review", {"code": "x = 1"})
- assert "x = 1" in str(prompt.messages[0].content), prompt
- print(f"ok pydantic-ai via {label}")
-
-
-async def main() -> None:
- import pydantic_ai
-
- print(f"fastmcp {fastmcp.__version__}, pydantic-ai {pydantic_ai.__version__}")
- await exercise("in-process FastMCP server", mcp)
- await exercise("stdio", str(SERVER))
- with http_server([sys.executable]) as url:
- await exercise("streamable HTTP", url)
+ command, args = stdio_command()
+ with report.transport("stdio"):
+ await report.check(
+ "installed build is under test", installed_build_is_under_test
+ )
+ await exercise(report, StdioTransport(command, args, env=server_env()))
+
+ with tempfile.TemporaryDirectory() as tmp:
+ config = Path(tmp) / "mcp.json"
+ config.write_text(
+ json.dumps(
+ {
+ "mcpServers": {
+ "smoke": {"command": command, "args": args, "env": server_env()}
+ }
+ }
+ )
+ )
+ with report.transport("mcp.json config"):
+
+ async def agent_loop_from_config() -> None:
+ agent = Agent(
+ FunctionModel(scripted_model), toolsets=load_mcp_toolsets(config)
+ )
+ result = await agent.run("add 2 and 3, then get the Chicago forecast")
+ assert "5" in result.output and "sunny" in result.output, result.output
+
+ await report.check(
+ "agent loop from load_mcp_toolsets", agent_loop_from_config
+ )
+
+ for transport, label in (("http", "streamable HTTP"), ("sse", "SSE")):
+ with report.transport(label), serve(transport) as server:
+
+ async def rejects_missing_token() -> None:
+ try:
+ async with MCPToolset(server.url):
+ pass
+ except Exception:
+ return
+ raise AssertionError("connected without a bearer token")
+
+ await report.check("rejects missing bearer token", rejects_missing_token)
+ await exercise(report, server.url, headers=server.headers)
+
+ report.finish()
if __name__ == "__main__":