π The fast, Pythonic way to build MCP servers and clients.
Run downstream smoke against built wheels and pydantic-ai's own MCP tests (#5209)
nate nowackf9f4ac892dd9
diff --git a/.github/workflows/run-downstream-smoke.yml b/.github/workflows/run-downstream-smoke.ymlindex 4fc01a049..8476d9b6b 100644--- a/.github/workflows/run-downstream-smoke.yml+++ b/.github/workflows/run-downstream-smoke.yml@@ -1,16 +1,25 @@name: Downstream smoke-# Drive this checkout's FastMCP through the frameworks that depend on it, using-# their latest releases, so a change that breaks them fails here first.+# Drive this checkout's FastMCP through the frameworks that depend on it, the+# way uv's ecosystem tests drive downstream projects: build once, install that+# build where the consumer runs, and run the consumer's own usage against it.+#+# Pushes and PRs use the pinned versions below, so a failure points at the+# change under test. The nightly run resolves each consumer's latest release+# to catch breakage that ships downstream.env:PY_COLORS: 1+ PYDANTIC_AI_VERSION: "2.47.0"+ LANGCHAIN_MCP_ADAPTERS_VERSION: "0.3.2"+ LANGCHAIN_VERSION: "1.4.2"on:push:branches: ["main"]paths:- "fastmcp_slim/**"+ - "fastmcp_tasks/**"- "tests/downstream/**"- "pyproject.toml"- "uv.lock"@@ -19,16 +28,21 @@ on:pull_request:paths:- "fastmcp_slim/**"+ - "fastmcp_tasks/**"- "tests/downstream/**"- "pyproject.toml"- "uv.lock"- ".github/workflows/run-downstream-smoke.yml"schedule:- # Downstream releases can break us without any change here.- cron: "30 2 * * *"workflow_dispatch:+ inputs:+ latest:+ description: "Test each consumer's latest release instead of the pins"+ type: boolean+ default: falseconcurrency:group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}@@ -38,31 +52,162 @@ permissions:contents: readjobs:+ plan:+ name: Plan+ runs-on: ubuntu-latest+ timeout-minutes: 5+ outputs:+ fastmcp: ${{ steps.build.outputs.version }}+ pydantic-ai: ${{ steps.versions.outputs.pydantic-ai }}+ langchain-mcp-adapters: ${{ steps.versions.outputs.langchain-mcp-adapters }}+ langchain: ${{ steps.versions.outputs.langchain }}+ steps:+ - uses: actions/checkout@v7+ with:+ fetch-depth: 0++ - name: Install uv+ uses: astral-sh/[email protected]++ - name: Build wheels+ id: build+ run: |+ for package in . fastmcp_slim fastmcp_tasks; do+ uv build --quiet --wheel --out-dir dist "$package"+ done+ version=$(basename dist/fastmcp_slim-*.whl | cut -d- -f2)+ echo "version=$version" >> "$GITHUB_OUTPUT"+ ls dist++ - uses: actions/upload-artifact@v7+ with:+ name: fastmcp-wheels+ path: dist/*.whl++ - name: Choose consumer versions+ id: versions+ env:+ LATEST: ${{ github.event_name == 'schedule' || inputs.latest == true }}+ run: |+ pick() {+ if [ "$LATEST" = true ]; then+ curl -fsS "https://pypi.org/pypi/$1/json" | jq -r .info.version+ else+ echo "$2"+ fi+ }+ {+ echo "pydantic-ai=$(pick pydantic-ai-slim "$PYDANTIC_AI_VERSION")"+ echo "langchain-mcp-adapters=$(pick langchain-mcp-adapters "$LANGCHAIN_MCP_ADAPTERS_VERSION")"+ echo "langchain=$(pick langchain "$LANGCHAIN_VERSION")"+ } | tee -a "$GITHUB_OUTPUT" > versions.txt+ {+ echo "### Downstream smoke plan"+ echo+ echo "FastMCP \`$(basename dist/fastmcp_slim-*.whl | cut -d- -f2)\` against $([ "$LATEST" = true ] && echo "each consumer's latest release" || echo "pinned consumer versions")"+ echo+ echo "| consumer | version |"+ echo "|---|---|"+ sed 's/^\(.*\)=\(.*\)$/| \1 | `\2` |/' versions.txt+ } >> "$GITHUB_STEP_SUMMARY"++ pydantic-ai-suite:+ name: pydantic-ai test suite+ needs: plan+ runs-on: ubuntu-latest+ timeout-minutes: 15+ steps:+ - uses: actions/checkout@v7+ with:+ path: fastmcp++ - uses: actions/checkout@v7+ with:+ repository: pydantic/pydantic-ai+ ref: v${{ needs.plan.outputs.pydantic-ai }}+ path: pydantic-ai+ persist-credentials: false++ - uses: actions/download-artifact@v8+ with:+ name: fastmcp-wheels+ path: dist++ - name: Install uv+ uses: astral-sh/[email protected]+ with:+ python-version: "3.13"+ enable-cache: true+ cache-dependency-glob: pydantic-ai/uv.lock++ # Mirrors pydantic-ai's own "test FastMCP 4 compatibility" job: their locked+ # environment, with FastMCP overlaid from the wheels under test.+ - name: Resolve environment+ working-directory: pydantic-ai+ run: |+ echo "UV_RUN=uv run --all-extras --no-extra mcp-tasks --all-packages \+ --with fastmcp@file://$(ls $GITHUB_WORKSPACE/dist/fastmcp-4*.whl) \+ --with fastmcp-slim@file://$(ls $GITHUB_WORKSPACE/dist/fastmcp_slim-*.whl) \+ --with fastmcp-tasks@file://$(ls $GITHUB_WORKSPACE/dist/fastmcp_tasks-*.whl)" >> "$GITHUB_ENV"++ - name: Check the build under test is installed+ working-directory: pydantic-ai+ run: >-+ $UV_RUN --quiet python -c "import fastmcp, mcp, importlib.metadata as m;+ assert fastmcp.__version__ == '${{ needs.plan.outputs.fastmcp }}', fastmcp.__version__;+ print('fastmcp', fastmcp.__version__, 'Β· mcp', m.version('mcp'))"++ - name: Run pydantic-ai's MCP tests+ working-directory: pydantic-ai+ run: >-+ $UV_RUN --quiet pytest -p no:cacheprovider -q+ --junitxml=$GITHUB_WORKSPACE/junit.xml+ tests/test_mcp.py tests/models/test_mcp_sampling.py tests/test_conftest.py++ - name: Summarize+ if: always() && hashFiles('junit.xml') != ''+ run: python3 fastmcp/tests/downstream/_harness.py junit.xml "pydantic-ai ${{ needs.plan.outputs.pydantic-ai }} MCP test suite"+smoke:- name: ${{ matrix.name }}+ name: ${{ matrix.consumer }} smoke+ needs: planruns-on: ubuntu-latesttimeout-minutes: 10strategy:fail-fast: falsematrix:include:- # pydantic-ai builds on fastmcp.Client, so it shares FastMCP's environment.- - name: pydantic-ai- run: uv run --with 'pydantic-ai-slim[mcp]' tests/downstream/smoke_pydantic_ai.py- # langchain-mcp-adapters pins mcp<2, so it runs in its own environment and reaches FastMCP over the wire.- - name: langchain- run: uv run tests/downstream/smoke_langchain.py-+ # Client-only, as `pydantic-ai-slim[mcp]` installs it, with the build under test.+ - consumer: pydantic-ai+ script: smoke_pydantic_ai.py+ with: >-+ --with "pydantic-ai-slim[mcp]==${{ needs.plan.outputs.pydantic-ai }}"+ --with "fastmcp-slim[client] @ file://$GITHUB_WORKSPACE/$(ls dist/fastmcp_slim-*.whl)"+ # The adapters pin mcp<2, so they reach FastMCP over the wire.+ - consumer: langchain+ script: smoke_langchain.py+ with: >-+ --with "langchain-mcp-adapters==${{ needs.plan.outputs.langchain-mcp-adapters }}"+ --with "langchain==${{ needs.plan.outputs.langchain }}"+ env:+ FASTMCP_EXPECTED_VERSION: ${{ needs.plan.outputs.fastmcp }}steps:- uses: actions/checkout@v7+ # The servers every smoke connects to run from this environment.- name: Setup uvuses: ./.github/actions/setup-uvwith:python-version: "3.12"- - name: Run ${{ matrix.name }} smoke- run: ${{ matrix.run }}- env:- PYDANTIC_AI_NO_BANNER: 1- FASTMCP_LOG_LEVEL: WARNING+ - uses: actions/download-artifact@v8+ with:+ name: fastmcp-wheels+ path: dist++ # Resolved outside the project and without its config, as a user's install is.+ - name: Resolve ${{ matrix.consumer }} environment+ run: uv run --isolated --no-project --no-config ${{ matrix.with }} python -c pass++ - name: Run ${{ matrix.consumer }} smoke+ run: uv run --quiet --isolated --no-project --no-config ${{ matrix.with }} python tests/downstream/${{ matrix.script }}diff --git a/tests/downstream/README.md b/tests/downstream/README.mdnew file mode 100644index 000000000..4ba87bbcf--- /dev/null+++ b/tests/downstream/README.md@@ -0,0 +1,35 @@+# downstream smoke++Checks that FastMCP still works the way its consumers use it. The+[workflow](../../.github/workflows/run-downstream-smoke.yml) builds FastMCP's+wheels once, installs them where each consumer runs, and drives the consumer+against servers started from the checkout.++| job | what it runs | FastMCP install |+|---|---|---|+| pydantic-ai test suite | pydantic-ai's own `tests/test_mcp.py` and sampling tests at a pinned tag | wheels overlaid on pydantic-ai's locked environment, as their FastMCP 4 job does |+| pydantic-ai smoke | `smoke_pydantic_ai.py` over stdio, `mcp.json` config, streamable HTTP and SSE | client-only `fastmcp-slim[client]`, as `pydantic-ai-slim[mcp]` installs it |+| langchain smoke | `smoke_langchain.py` over stdio, streamable HTTP and SSE | none: the adapters pin `mcp<2`, so they reach FastMCP over the wire |++HTTP and SSE servers require a bearer token. Each smoke prints one line per+check, writes a table to the job summary, and reports warnings raised in the+consumer's process.++Pushes and PRs test the versions pinned in the workflow. The nightly run, or a+manual run with `latest`, tests each consumer's latest release.++## running locally++```bash+uv sync+uv build --wheel --out-dir dist fastmcp_slim++uv run --isolated --no-project --no-config \+ --with 'pydantic-ai-slim[mcp]' \+ --with "fastmcp-slim[client] @ file://$PWD/$(ls dist/fastmcp_slim-*.whl)" \+ python tests/downstream/smoke_pydantic_ai.py++uv run --isolated --no-project --no-config \+ --with langchain-mcp-adapters --with langchain \+ python tests/downstream/smoke_langchain.py+```diff --git a/tests/downstream/_harness.py b/tests/downstream/_harness.pynew file mode 100644index 000000000..19f58e902--- /dev/null+++ b/tests/downstream/_harness.py@@ -0,0 +1,216 @@+"""Shared plumbing for the downstream smoke scripts: server processes and reporting."""++import os+import secrets+import socket+import subprocess+import sys+import time+import traceback+import warnings+from collections.abc import Awaitable, Callable, Iterator+from contextlib import contextmanager+from dataclasses import dataclass, field+from pathlib import Path++HERE = Path(__file__).parent+SERVER = HERE / "server.py"+REPO = HERE.parents[1]+INSTRUCTIONS = "Arithmetic and weather for smoke tests."+IN_ACTIONS = os.environ.get("GITHUB_ACTIONS") == "true"+os.environ.setdefault("PYDANTIC_AI_NO_BANNER", "1")+++def server_python() -> str:+ """The interpreter of the FastMCP checkout, which is what serves every smoke test."""+ return os.environ.get("FASTMCP_SERVER_PYTHON") or str(+ REPO / ".venv" / "bin" / "python"+ )+++def server_env() -> dict[str, str]:+ """Keep the server's own logs and warnings out of the consumer's output."""+ return {**os.environ, "FASTMCP_LOG_LEVEL": "CRITICAL", "PYTHONWARNINGS": "ignore"}+++def stdio_command() -> tuple[str, list[str]]:+ return server_python(), [str(SERVER), "stdio"]+++@dataclass+class Server:+ url: str+ token: str++ @property+ def headers(self) -> dict[str, str]:+ return {"Authorization": f"Bearer {self.token}"}+++@contextmanager+def serve(transport: str) -> Iterator[Server]:+ """Run server.py over `http` or `sse` with bearer auth for the length of the block."""+ with socket.socket() as sock:+ sock.bind(("127.0.0.1", 0))+ port = sock.getsockname()[1]+ token = secrets.token_urlsafe(16)+ proc = subprocess.Popen(+ [server_python(), str(SERVER), transport, str(port), token], env=server_env()+ )+ try:+ deadline = time.monotonic() + 30+ while True:+ if proc.poll() is not None:+ raise RuntimeError(f"{transport} server exited with {proc.returncode}")+ try:+ socket.create_connection(("127.0.0.1", port), timeout=0.2).close()+ break+ except OSError:+ if time.monotonic() > deadline:+ raise RuntimeError(+ f"{transport} server did not listen within 30s"+ ) from None+ time.sleep(0.1)+ path = "/sse" if transport == "sse" else "/mcp"+ yield Server(url=f"http://127.0.0.1:{port}{path}", token=token)+ finally:+ proc.terminate()+ proc.wait(timeout=10)+++@dataclass+class Result:+ transport: str+ check: str+ error: str | None+ seconds: float+++@dataclass+class Report:+ consumer: str+ versions: dict[str, str]+ results: list[Result] = field(default_factory=list)+ warnings: dict[str, str] = field(default_factory=dict)+ _transport: str = ""++ @contextmanager+ def transport(self, name: str) -> Iterator[None]:+ self._transport = name+ print(+ f"::group::{self.consumer} via {name}"+ if IN_ACTIONS+ else f"\n{self.consumer} via {name}"+ )+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ try:+ yield+ except Exception as error:+ self._record("connect", error, 0.0)+ for w in caught:+ if issubclass(w.category, (DeprecationWarning, UserWarning)):+ self.warnings.setdefault(f"{w.category.__name__}: {w.message}", name)+ if IN_ACTIONS:+ print("::endgroup::")++ async def check(self, name: str, fn: Callable[[], Awaitable[object]]) -> None:+ start = time.monotonic()+ try:+ await fn()+ error = None+ except Exception as exc:+ error = exc+ self._record(name, error, time.monotonic() - start)++ def _record(self, name: str, error: BaseException | None, seconds: float) -> None:+ detail = None if error is None else f"{type(error).__name__}: {error}"+ self.results.append(Result(self._transport, name, detail, seconds))+ mark = "β" if error is None else "β"+ print(+ f" {mark} {name:<34} {seconds * 1000:6.0f} ms"+ + ("" if error is None else f"\n {detail}")+ )+ if error is not None:+ traceback.print_exception(error, file=sys.stdout)++ def finish(self) -> None:+ failed = [r for r in self.results if r.error]+ transports = list(dict.fromkeys(r.transport for r in self.results))+ checks = list(dict.fromkeys(r.check for r in self.results))+ cell = {+ (r.transport, r.check): ("β " if not r.error else "β")+ for r in self.results+ }++ lines = [+ f"### {'β' if failed else 'β '} {self.consumer}",+ "",+ " Β· ".join(f"`{k} {v}`" for k, v in self.versions.items()),+ "",+ "| check | " + " | ".join(transports) + " |",+ "|---|" + "---|" * len(transports),+ *(+ f"| {c} | "+ + " | ".join(cell.get((t, c), "β") for t in transports)+ + " |"+ for c in checks+ ),+ ]+ if failed:+ lines += ["", "**Failures**", ""]+ lines += [f"- `{r.transport}` / `{r.check}`: {r.error}" for r in failed]+ if self.warnings:+ lines += [+ "",+ "<details><summary>Warnings raised in the consumer's process</summary>",+ "",+ ]+ lines += [+ f"- `{where}`: {message}" for message, where in self.warnings.items()+ ]+ lines += ["", "</details>"]+ summary = "\n".join(lines) + "\n"++ print(f"\n{len(self.results) - len(failed)}/{len(self.results)} checks passed")+ for message, where in self.warnings.items():+ print(+ f"::warning title={self.consumer} via {where}::{message}"+ if IN_ACTIONS+ else f"warning via {where}: {message}"+ )+ if path := os.environ.get("GITHUB_STEP_SUMMARY"):+ with open(path, "a") as fh:+ fh.write(summary)+ if failed:+ sys.exit(1)+++def junit_summary(path: str, title: str) -> None:+ """Append a pass/fail line and any failures from a pytest JUnit report to the job summary."""+ import xml.etree.ElementTree as ET++ suite = ET.parse(path).getroot()+ suite = suite if suite.tag == "testsuite" else suite[0]+ total, failures, errors, skipped = (+ int(suite.get(k, 0)) for k in ("tests", "failures", "errors", "skipped")+ )+ failed = [+ f"- `{case.get('classname')}::{case.get('name')}`"+ for case in suite.iter("testcase")+ if case.find("failure") is not None or case.find("error") is not None+ ]+ mark = "β" if failed else "β "+ lines = [+ f"### {mark} {title}",+ "",+ f"{total - failures - errors - skipped} passed, {failures + errors} failed, {skipped} skipped",+ *(["", *failed] if failed else []),+ ]+ if summary := os.environ.get("GITHUB_STEP_SUMMARY"):+ with open(summary, "a") as fh:+ fh.write("\n".join(lines) + "\n")+++if __name__ == "__main__":+ junit_summary(sys.argv[1], sys.argv[2])diff --git a/tests/downstream/_http.py b/tests/downstream/_http.pydeleted file mode 100644index 70f420db5..000000000--- a/tests/downstream/_http.py+++ /dev/null@@ -1,40 +0,0 @@-"""Serve server.py over streamable HTTP in a subprocess for the length of a `with` block."""--import os-import socket-import subprocess-import sys-import time-from collections.abc import Iterator-from contextlib import contextmanager-from pathlib import Path--SERVER = Path(__file__).with_name("server.py")---def free_port() -> int:- with socket.socket() as sock:- sock.bind(("127.0.0.1", 0))- return sock.getsockname()[1]---@contextmanager-def http_server(python: list[str]) -> Iterator[str]:- port = free_port()- proc = subprocess.Popen([*python, str(SERVER), "http", str(port)], env=os.environ)- try:- deadline = time.monotonic() + 30- while time.monotonic() < deadline:- if proc.poll() is not None:- sys.exit(f"server exited with {proc.returncode}")- try:- socket.create_connection(("127.0.0.1", port), timeout=0.2).close()- break- except OSError:- time.sleep(0.1)- else:- sys.exit("server did not start listening within 30s")- yield f"http://127.0.0.1:{port}/mcp"- finally:- proc.terminate()- proc.wait(timeout=10)diff --git a/tests/downstream/server.py b/tests/downstream/server.pyindex 0cf931cf8..e28ee8f3d 100644--- a/tests/downstream/server.py+++ b/tests/downstream/server.py@@ -1,117 +1,176 @@"""The FastMCP server every downstream smoke script connects to.-Run as `python server.py stdio` or `python server.py http <port>`.+python server.py stdio+python server.py http <port> <bearer-token>+python server.py sse <port> <bearer-token>"""+import base64import sysfrom dataclasses import dataclass+from _harness import INSTRUCTIONSfrom mcp.types import (ElicitRequest,ElicitRequestFormParams,ElicitResult,+ EmbeddedResource,InputRequiredResult,+ ResourceLink,+ TextContent,+ TextResourceContents,)from mcp_types.version import MODERN_PROTOCOL_VERSIONSfrom pydantic import BaseModelfrom fastmcp import Context, FastMCPfrom fastmcp.exceptions import ToolError+from fastmcp.server.auth.providers.jwt import StaticTokenVerifier+from fastmcp.utilities.types import Audio, Image-mcp = FastMCP(- "downstream-smoke", instructions="Arithmetic and weather for smoke tests."+PNG = base64.b64decode(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="+)+WAV = (+ b"RIFF&\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00D\xac\x00\x00"+ b"\x88X\x01\x00\x02\x00\x10\x00data\x02\x00\x00\x00\x00\x00")-class Forecast(BaseModel):- city: str- celsius: float- conditions: list[str]---def add(a: int, b: int) -> int:- """Add two integers."""- return a + b---def forecast(city: str, days: int = 1) -> Forecast:- """Structured weather forecast for a city."""- return Forecast(city=city, celsius=21.5, conditions=["sunny"] * days)---def divide(a: float, b: float) -> float:- """Divide a by b."""- if b == 0:- raise ToolError("cannot divide by zero")- return a / b---async def count_to(n: int, ctx: Context) -> str:- """Count to n, reporting progress and logging along the way."""- for i in range(1, n + 1):- await ctx.report_progress(i, n)- await ctx.info(f"counted to {n}")- return f"counted to {n}"---@dataclass-class Approval:- approved: bool---async def confirm(action: str, ctx: Context) -> str | InputRequiredResult:- """Ask the client to approve an action, using whichever elicitation path the connection's protocol era supports."""- rc = ctx.request_context- if rc is not None and rc.protocol_version in MODERN_PROTOCOL_VERSIONS:- if ctx.input_responses is None:- form = ElicitRequestFormParams(- message=f"approve {action}?",- requested_schema={- "type": "object",- "properties": {"approved": {"type": "boolean"}},- "required": ["approved"],- },- )- return InputRequiredResult(- result_type="input_required",- input_requests={- "approval": ElicitRequest(method="elicitation/create", params=form)- },- )- answer = ctx.input_responses["approval"]- assert isinstance(answer, ElicitResult), answer- if answer.action == "accept" and answer.content and answer.content["approved"]:+def build(auth_token: str | None = None) -> FastMCP:+ auth = (+ StaticTokenVerifier({auth_token: {"client_id": "smoke", "scopes": []}})+ if auth_token+ else None+ )+ mcp = FastMCP("downstream-smoke", instructions=INSTRUCTIONS, auth=auth)++ class Forecast(BaseModel):+ city: str+ celsius: float+ conditions: list[str]++ @dataclass+ class Approval:+ approved: bool++ @mcp.tool+ def add(a: int, b: int) -> int:+ """Add two integers."""+ return a + b++ @mcp.tool+ def forecast(city: str, days: int = 1) -> Forecast:+ """Structured weather forecast for a city."""+ return Forecast(city=city, celsius=21.5, conditions=["sunny"] * days)++ @mcp.tool+ def divide(a: float, b: float) -> float:+ """Divide a by b."""+ if b == 0:+ raise ToolError("cannot divide by zero")+ return a / b++ @mcp.tool+ async def count_to(n: int, ctx: Context) -> str:+ """Count to n, reporting progress and logging along the way."""+ for i in range(1, n + 1):+ await ctx.report_progress(i, n)+ await ctx.info(f"counted to {n}")+ return f"counted to {n}"++ @mcp.tool+ def snapshot() -> Image:+ """A 1x1 PNG."""+ return Image(data=PNG, format="png")++ @mcp.tool+ def chime() -> Audio:+ """A silent WAV."""+ return Audio(data=WAV, format="wav")++ @mcp.tool+ def attachments() -> list:+ """Mixed content: text, an embedded resource, and a resource link."""+ return [+ TextContent(type="text", text="see attached"),+ EmbeddedResource(+ type="resource",+ resource=TextResourceContents(+ uri="notes://today", mime_type="text/plain", text="buy milk"+ ),+ ),+ ResourceLink(+ type="resource_link",+ uri="config://app",+ name="app config",+ mime_type="application/json",+ ),+ ]++ @mcp.tool+ async def confirm(action: str, ctx: Context) -> str | InputRequiredResult:+ """Ask the client to approve an action, using whichever elicitation path the connection's protocol era supports."""+ rc = ctx.request_context+ if rc is not None and rc.protocol_version in MODERN_PROTOCOL_VERSIONS:+ if ctx.input_responses is None:+ form = ElicitRequestFormParams(+ message=f"approve {action}?",+ requested_schema={+ "type": "object",+ "properties": {"approved": {"type": "boolean"}},+ "required": ["approved"],+ },+ )+ return InputRequiredResult(+ result_type="input_required",+ input_requests={+ "approval": ElicitRequest(+ method="elicitation/create", params=form+ )+ },+ )+ answer = ctx.input_responses["approval"]+ assert isinstance(answer, ElicitResult), answer+ if (+ answer.action == "accept"+ and answer.content+ and answer.content["approved"]+ ):+ return f"{action}: approved"+ return f"{action}: {answer.action}"+ result = await ctx.elicit(f"approve {action}?", response_type=Approval)+ if result.action == "accept" and result.data.approved:return f"{action}: approved"- return f"{action}: {answer.action}"- result = await ctx.elicit(f"approve {action}?", response_type=Approval)- if result.action == "accept" and result.data.approved:- return f"{action}: approved"- return f"{action}: {result.action}"-+ return f"{action}: {result.action}"[email protected]("config://app", mime_type="application/json")-def app_config() -> str:- return '{"mode": "smoke"}'+ @mcp.resource("config://app", mime_type="application/json")+ def app_config() -> str:+ return '{"mode": "smoke"}'+ @mcp.resource("greeting://{name}")+ def greeting(name: str) -> str:+ return f"hello, {name}"[email protected]("greeting://{name}")-def greeting(name: str) -> str:- return f"hello, {name}"+ @mcp.prompt+ def review(code: str) -> str:+ """Ask for a code review."""+ return f"please review:\n{code}"--def review(code: str) -> str:- """Ask for a code review."""- return f"please review:\n{code}"+ return mcpif __name__ == "__main__":- if sys.argv[1:2] == ["http"]:- mcp.run("http", host="127.0.0.1", port=int(sys.argv[2]), show_banner=False)- else:- mcp.run("stdio", show_banner=False)+ match sys.argv[1:]:+ case [] | ["stdio"]:+ build().run("stdio", show_banner=False)+ case ["http" | "sse" as transport, port, token]:+ build(token).run(+ transport,+ host="127.0.0.1",+ port=int(port),+ show_banner=False,+ log_level="critical",+ )+ case args:+ sys.exit(f"usage: server.py [stdio | http|sse <port> <token>], got {args}")diff --git a/tests/downstream/smoke_langchain.py b/tests/downstream/smoke_langchain.pyindex d070bac36..d4bb32917 100644--- a/tests/downstream/smoke_langchain.py+++ b/tests/downstream/smoke_langchain.py@@ -1,30 +1,36 @@-# /// script-# requires-python = ">=3.10"-# dependencies = ["langchain-mcp-adapters", "langchain"]-# ///-"""Exercise a FastMCP server through langchain-mcp-adapters.+"""Drive a FastMCP server through langchain-mcp-adapters, the way LangChain users reach it.The adapters pin `mcp<2` while FastMCP 4 needs `mcp>=2`, so they cannot share an-environment. This script runs in its own environment and reaches the server-over the wire, the way LangChain users do:+environment. This runs in its own environment against servers started from the+checkout, which also covers handshake-era (mcp v1) clients:- uv run tests/downstream/smoke_langchain.py+ uv run --isolated --no-project --with langchain-mcp-adapters --with langchain \+ python tests/downstream/smoke_langchain.py"""import asyncio+from importlib.metadata import versionfrom typing import Any-from _http import SERVER, http_server+from _harness import INSTRUCTIONS, Report, serve, server_env, stdio_commandfrom langchain.agents import create_agentfrom langchain_core.language_models.fake_chat_models import GenericFakeChatModel-from langchain_core.messages import AIMessage+from langchain_core.messages import AIMessage, ToolMessagefrom langchain_mcp_adapters.callbacks import Callbacksfrom langchain_mcp_adapters.client import MultiServerMCPClientfrom langchain_mcp_adapters.tools import load_mcp_toolsfrom mcp.types import ElicitResult-REPO = SERVER.parents[2]-FASTMCP_PYTHON = ["uv", "run", "--project", str(REPO), "--frozen", "python"]+TOOLS = {+ "add",+ "forecast",+ "divide",+ "count_to",+ "snapshot",+ "chime",+ "attachments",+ "confirm",+}class ScriptedToolModel(GenericFakeChatModel):@@ -53,7 +59,14 @@ def scripted_model() -> ScriptedToolModel:)-async def exercise(label: str, connection: dict[str, Any]) -> None:+async def call(tool: Any, args: dict[str, Any]) -> ToolMessage:+ """Invoke a tool the way an agent does, so the result is a ToolMessage with its artifact."""+ return await tool.ainvoke(+ {"type": "tool_call", "name": tool.name, "args": args, "id": "call-1"}+ )+++async def exercise(report: Report, connection: dict[str, Any]) -> None:progress: list[float] = []logs: list[str] = []@@ -78,70 +91,150 @@ async def exercise(label: str, connection: dict[str, Any]) -> None:on_elicitation=on_elicitation,),)+ tools: dict[str, Any] = {}- tools = {tool.name: tool for tool in await client.get_tools()}- assert {"add", "forecast", "divide", "count_to", "confirm"} <= set(tools), tools+ async def lists_tools() -> None:+ tools.update({tool.name: tool for tool in await client.get_tools()})+ assert TOOLS <= set(tools), sorted(tools)- agent = create_agent(scripted_model(), list(tools.values()))- state = await agent.ainvoke(- {"messages": [("user", "add 2 and 3, then get the Chicago forecast")]}- )- returned = {m.name: m.content for m in state["messages"] if m.type == "tool"}- assert "5" in str(returned["add"]), returned- assert "sunny" in str(returned["forecast"]), returned+ async def server_info() -> None:+ info = (await client.get_server_info(server_name="smoke"))["smoke"]+ assert info.instructions == INSTRUCTIONS, info.instructions+ assert info.serverInfo.name == "downstream-smoke", info.serverInfo- assert "42" in str(await tools["add"].ainvoke({"a": 40, "b": 2}))- failed = await tools["divide"].ainvoke(- {- "type": "tool_call",- "name": "divide",- "args": {"a": 1, "b": 0},- "id": "call-divide",- }- )- assert failed.status == "error" and "divide by zero" in str(failed.content), failed+ async def agent_loop() -> None:+ agent = create_agent(scripted_model(), list(tools.values()))+ state = await agent.ainvoke(+ {"messages": [("user", "add 2 and 3, then get the Chicago forecast")]}+ )+ returned = {m.name: m.content for m in state["messages"] if m.type == "tool"}+ assert "5" in str(returned["add"]) and "sunny" in str(returned["forecast"]), (+ returned+ )- assert "deploy: approved" in str(- await tools["confirm"].ainvoke({"action": "deploy"})- )- assert "counted to 3" in str(await tools["count_to"].ainvoke({"n": 3}))- assert progress == [1, 2, 3], progress- assert any("counted to 3" in line for line in logs), logs+ async def structured_output() -> None:+ message = await call(tools["forecast"], {"city": "Oslo"})+ expected = {"city": "Oslo", "celsius": 21.5, "conditions": ["sunny"]}+ assert message.artifact == {"structured_content": expected}, message.artifact- async with client.session("smoke") as session:- assert {"add", "forecast"} <= {- tool.name for tool in await load_mcp_tools(session)- }+ async def tool_error() -> None:+ message = await call(tools["divide"], {"a": 1, "b": 0})+ assert message.status == "error" and "divide by zero" in str(message.content), (+ message+ )- blobs = await client.get_resources(- "smoke", uris=["config://app", "greeting://nate"]- )- assert [blob.as_string() for blob in blobs] == [- '{"mode": "smoke"}',- "hello, nate",- ], blobs+ async def image() -> None:+ [block] = (await call(tools["snapshot"], {})).content+ assert (+ block["type"] == "image"+ and block["mime_type"] == "image/png"+ and block["base64"]+ ), block++ async def mixed_content() -> None:+ blocks = (await call(tools["attachments"], {})).content+ shapes = [(b["type"], b.get("text") or b.get("url")) for b in blocks]+ assert shapes == [+ ("text", "see attached"),+ ("text", "buy milk"),+ ("file", "config://app"),+ ], shapes++ async def elicitation() -> None:+ assert "deploy: approved" in str(+ (await call(tools["confirm"], {"action": "deploy"})).content+ )- messages = await client.get_prompt("smoke", "review", arguments={"code": "x = 1"})- assert "x = 1" in str(messages[0].content), messages- print(f"ok langchain via {label}")+ async def progress_and_logging() -> None:+ assert "counted to 3" in str((await call(tools["count_to"], {"n": 3})).content)+ assert progress == [1, 2, 3], progress+ assert any("counted to 3" in line for line in logs), logs++ async def explicit_session() -> None:+ async with client.session("smoke") as session:+ assert TOOLS <= {tool.name for tool in await load_mcp_tools(session)}++ async def resources() -> None:+ [blob] = await client.get_resources("smoke", uris=["config://app"])+ assert (+ blob.as_string() == '{"mode": "smoke"}'+ and blob.mimetype == "application/json"+ ), blob++ async def resource_template() -> None:+ [blob] = await client.get_resources("smoke", uris=["greeting://nate"])+ assert blob.as_string() == "hello, nate", blob++ async def prompt() -> None:+ messages = await client.get_prompt(+ "smoke", "review", arguments={"code": "x = 1"}+ )+ assert "x = 1" in str(messages[0].content), messages++ for check in (+ lists_tools,+ server_info,+ agent_loop,+ structured_output,+ tool_error,+ image,+ mixed_content,+ elicitation,+ progress_and_logging,+ explicit_session,+ resources,+ resource_template,+ prompt,+ ):+ await report.check(check.__name__.replace("_", " "), check)async def main() -> None:- from importlib.metadata import version-- print(- f"langchain-mcp-adapters {version('langchain-mcp-adapters')}, mcp {version('mcp')}"- )- await exercise(- "stdio",+ report = Report(+ "langchain",{- "transport": "stdio",- "command": FASTMCP_PYTHON[0],- "args": [*FASTMCP_PYTHON[1:], str(SERVER), "stdio"],+ "langchain-mcp-adapters": version("langchain-mcp-adapters"),+ "langchain": version("langchain"),+ "mcp": version("mcp"),},)- with http_server(FASTMCP_PYTHON) as url:- await exercise("streamable HTTP", {"transport": "streamable_http", "url": url})+ command, args = stdio_command()+ with report.transport("stdio"):+ await exercise(+ report,+ {+ "transport": "stdio",+ "command": command,+ "args": args,+ "env": server_env(),+ },+ )++ for transport, label, kind in (+ ("http", "streamable HTTP", "streamable_http"),+ ("sse", "SSE", "sse"),+ ):+ with report.transport(label), serve(transport) as server:++ async def rejects_missing_token() -> None:+ client = MultiServerMCPClient(+ {"smoke": {"transport": kind, "url": server.url}}+ )+ try:+ await client.get_tools()+ except BaseException as error:+ if isinstance(error, KeyboardInterrupt):+ raise+ return+ raise AssertionError("listed tools without a bearer token")++ await report.check("rejects missing bearer token", rejects_missing_token)+ await exercise(+ report,+ {"transport": kind, "url": server.url, "headers": server.headers},+ )++ report.finish()if __name__ == "__main__":diff --git a/tests/downstream/smoke_pydantic_ai.py b/tests/downstream/smoke_pydantic_ai.pyindex a8f7494b5..a0d0d2573 100644--- a/tests/downstream/smoke_pydantic_ai.py+++ b/tests/downstream/smoke_pydantic_ai.py@@ -1,17 +1,26 @@-"""Exercise FastMCP through pydantic-ai's MCPToolset, which is built on fastmcp.Client.+"""Drive FastMCP through pydantic-ai's MCPToolset, as a pydantic-ai user installs it.-Runs inside the FastMCP project environment with pydantic-ai layered on top:+`pydantic-ai-slim[mcp]` depends on the client-only `fastmcp-slim[client]`, so+this runs in its own environment with no FastMCP server dependencies, resolved+together with the build under test. The servers run from the checkout:- uv run --with 'pydantic-ai-slim[mcp]' tests/downstream/smoke_pydantic_ai.py+ uv build --wheel --out-dir dist fastmcp_slim+ uv run --isolated --no-project --with 'pydantic-ai-slim[mcp]' \+ --with "fastmcp-slim[client] @ file://$PWD/$(ls dist/fastmcp_slim-*.whl)" \+ python tests/downstream/smoke_pydantic_ai.py"""import asyncio-import sys+import json+import os+import tempfile+from importlib.metadata import version+from pathlib import Pathfrom typing import Any-from _http import SERVER, http_server-from pydantic_ai import Agent, ModelRetry-from pydantic_ai.mcp import MCPToolset+from _harness import INSTRUCTIONS, Report, serve, server_env, stdio_command+from pydantic_ai import Agent, BinaryContent, ModelRetry+from pydantic_ai.mcp import MCPToolset, load_mcp_toolsetsfrom pydantic_ai.messages import (ModelMessage,ModelResponse,@@ -20,9 +29,20 @@ from pydantic_ai.messages import (ToolReturnPart,)from pydantic_ai.models.function import AgentInfo, FunctionModel-from server import mcpimport fastmcp+from fastmcp.client.transports import StdioTransport++TOOLS = {+ "add",+ "forecast",+ "divide",+ "count_to",+ "snapshot",+ "chime",+ "attachments",+ "confirm",+}def scripted_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse:@@ -34,10 +54,13 @@ def scripted_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponif isinstance(p, ToolReturnPart)]if not returns:+ # load_mcp_toolsets prefixes tool names with the server's config key.+ name = {t.name.rsplit("_", 1)[-1]: t.name for t in info.function_tools}+ name.update({t.name: t.name for t in info.function_tools})return ModelResponse(parts=[- ToolCallPart("add", {"a": 2, "b": 3}),- ToolCallPart("forecast", {"city": "Chicago", "days": 2}),+ ToolCallPart(name["add"], {"a": 2, "b": 3}),+ ToolCallPart(name["forecast"], {"city": "Chicago", "days": 2}),])return ModelResponse(parts=[TextPart(" | ".join(str(p.content) for p in returns))])@@ -49,7 +72,9 @@ async def accept(return {"approved": True}-async def exercise(label: str, target: Any) -> None:+async def exercise(+ report: Report, target: Any, headers: dict[str, str] | None = None+) -> None:progress: list[float] = []logs: list[str] = []@@ -63,57 +88,166 @@ async def exercise(label: str, target: Any) -> None:toolset = MCPToolset(target,+ headers=headers,+ include_instructions=True,elicitation_handler=accept,progress_handler=on_progress,log_handler=on_log,)async with toolset:- names = {tool.name for tool in await toolset.list_tools()}- assert {"add", "forecast", "divide", "count_to", "confirm"} <= names, names- agent = Agent(FunctionModel(scripted_model), toolsets=[toolset])- result = await agent.run("add 2 and 3, then get the Chicago forecast")- assert "5" in result.output and "sunny" in result.output, result.output+ async def lists_tools() -> None:+ names = {tool.name for tool in await toolset.list_tools()}+ assert TOOLS <= names, names++ async def instructions() -> None:+ assert toolset.instructions == INSTRUCTIONS, toolset.instructions++ async def agent_loop() -> None:+ agent = Agent(FunctionModel(scripted_model), toolsets=[toolset])+ result = await agent.run("add 2 and 3, then get the Chicago forecast")+ assert "5" in result.output and "sunny" in result.output, result.output++ async def structured_output() -> None:+ forecast = await toolset.direct_call_tool("forecast", {"city": "Oslo"})+ assert forecast == {+ "city": "Oslo",+ "celsius": 21.5,+ "conditions": ["sunny"],+ }, forecast++ async def tool_error() -> None:+ try:+ await toolset.direct_call_tool("divide", {"a": 1, "b": 0})+ except ModelRetry as error:+ assert "divide by zero" in str(error), error+ else:+ raise AssertionError("divide by zero did not raise ModelRetry")++ async def image() -> None:+ result = await toolset.direct_call_tool("snapshot", {})+ assert (+ isinstance(result, BinaryContent) and result.media_type == "image/png"+ ), result++ async def audio() -> None:+ result = await toolset.direct_call_tool("chime", {})+ assert (+ isinstance(result, BinaryContent) and result.media_type == "audio/wav"+ ), result++ async def mixed_content() -> None:+ result = await toolset.direct_call_tool("attachments", {})+ assert result == ["see attached", "buy milk", "config://app"], result++ async def elicitation() -> None:+ result = await toolset.direct_call_tool("confirm", {"action": "deploy"})+ assert result == "deploy: approved", result++ async def progress_and_logging() -> None:+ assert (+ await toolset.direct_call_tool("count_to", {"n": 3}) == "counted to 3"+ )+ assert progress == [1, 2, 3], progress+ assert any("counted to 3" in line for line in logs), logs++ async def resources() -> None:+ assert "config://app" in {+ str(r.uri) for r in await toolset.list_resources()+ }+ assert await toolset.read_resource("config://app") == '{"mode": "smoke"}'++ async def resource_template() -> None:+ templates = {+ t.uri_template for t in await toolset.list_resource_templates()+ }+ assert "greeting://{name}" in templates, templates+ assert await toolset.read_resource("greeting://nate") == "hello, nate"++ async def prompt() -> None:+ result = await toolset.get_prompt("review", {"code": "x = 1"})+ assert "x = 1" in str(result.messages[0].content), result++ for check in (+ lists_tools,+ instructions,+ agent_loop,+ structured_output,+ tool_error,+ image,+ audio,+ mixed_content,+ elicitation,+ progress_and_logging,+ resources,+ resource_template,+ prompt,+ ):+ await report.check(check.__name__.replace("_", " "), check)- assert await toolset.direct_call_tool("add", {"a": 40, "b": 2}) == 42- forecast = await toolset.direct_call_tool("forecast", {"city": "Oslo"})- assert forecast == {"city": "Oslo", "celsius": 21.5, "conditions": ["sunny"]}, (- forecast- )- try:- await toolset.direct_call_tool("divide", {"a": 1, "b": 0})- except ModelRetry as error:- assert "divide by zero" in str(error), error- else:- raise AssertionError("divide by zero did not raise")+async def main() -> None:+ report = Report(+ "pydantic-ai",+ {+ "fastmcp": fastmcp.__version__,+ "pydantic-ai-slim": version("pydantic-ai-slim"),+ "mcp": version("mcp"),+ },+ )- assert (- await toolset.direct_call_tool("confirm", {"action": "deploy"})- == "deploy: approved"+ async def installed_build_is_under_test() -> None:+ expected = os.environ.get("FASTMCP_EXPECTED_VERSION")+ assert expected in (None, fastmcp.__version__), (+ f"expected {expected}, got {fastmcp.__version__}")- assert await toolset.direct_call_tool("count_to", {"n": 3}) == "counted to 3"- assert progress == [1, 2, 3], progress- assert any("counted to 3" in line for line in logs), logs-- assert "config://app" in {str(r.uri) for r in await toolset.list_resources()}- assert await toolset.read_resource("config://app") == '{"mode": "smoke"}'- assert await toolset.read_resource("greeting://nate") == "hello, nate"-- prompt = await toolset.get_prompt("review", {"code": "x = 1"})- assert "x = 1" in str(prompt.messages[0].content), prompt- print(f"ok pydantic-ai via {label}")---async def main() -> None:- import pydantic_ai-- print(f"fastmcp {fastmcp.__version__}, pydantic-ai {pydantic_ai.__version__}")- await exercise("in-process FastMCP server", mcp)- await exercise("stdio", str(SERVER))- with http_server([sys.executable]) as url:- await exercise("streamable HTTP", url)+ command, args = stdio_command()+ with report.transport("stdio"):+ await report.check(+ "installed build is under test", installed_build_is_under_test+ )+ await exercise(report, StdioTransport(command, args, env=server_env()))++ with tempfile.TemporaryDirectory() as tmp:+ config = Path(tmp) / "mcp.json"+ config.write_text(+ json.dumps(+ {+ "mcpServers": {+ "smoke": {"command": command, "args": args, "env": server_env()}+ }+ }+ )+ )+ with report.transport("mcp.json config"):++ async def agent_loop_from_config() -> None:+ agent = Agent(+ FunctionModel(scripted_model), toolsets=load_mcp_toolsets(config)+ )+ result = await agent.run("add 2 and 3, then get the Chicago forecast")+ assert "5" in result.output and "sunny" in result.output, result.output++ await report.check(+ "agent loop from load_mcp_toolsets", agent_loop_from_config+ )++ for transport, label in (("http", "streamable HTTP"), ("sse", "SSE")):+ with report.transport(label), serve(transport) as server:++ async def rejects_missing_token() -> None:+ try:+ async with MCPToolset(server.url):+ pass+ except Exception:+ return+ raise AssertionError("connected without a bearer token")++ await report.check("rejects missing bearer token", rejects_missing_token)+ await exercise(report, server.url, headers=server.headers)++ report.finish()if __name__ == "__main__":