Skip to content

Personal AI, On Personal Devices

Python

fix(mcp): validate response id in StdioTransport.send() (#798)

havolline194bdb402225b
diff --git a/src/openjarvis/mcp/transport.py b/src/openjarvis/mcp/transport.py
index bfeb1b88..415e5112 100644
--- a/src/openjarvis/mcp/transport.py
+++ b/src/openjarvis/mcp/transport.py
@@ -2,9 +2,12 @@
from __future__ import annotations
+import json
import logging
+import queue
import subprocess
import threading
+import time
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Any, List, Optional
@@ -62,10 +65,31 @@ class StdioTransport(MCPTransport):
stdin/stdout.
"""
- def __init__(self, command: List[str]) -> None:
+ _STDOUT_QUEUE_SIZE = 1024
+ _STDOUT_EOF = object()
+
+ def __init__(
+ self,
+ command: List[str],
+ *,
+ response_timeout: float = 600.0,
+ ) -> None:
+ if response_timeout <= 0:
+ raise ValueError("response_timeout must be positive")
self._command = command
+ self._response_timeout = response_timeout
self._process: Optional[subprocess.Popen[str]] = None
+ self._stdout_queue: queue.Queue[Any] = queue.Queue(
+ maxsize=self._STDOUT_QUEUE_SIZE
+ )
+ self._reader_stop = threading.Event()
+ self._reader_thread: Optional[threading.Thread] = None
self._stderr_thread: Optional[threading.Thread] = None
+ # A single stdout stream cannot safely serve multiple independent
+ # readers: one request could consume and discard another request's
+ # response. Serialize complete write/read exchanges (and notification
+ # writes) so response correlation remains lossless.
+ self._request_lock = threading.Lock()
self._start()
def _start(self) -> None:
@@ -77,17 +101,68 @@ class StdioTransport(MCPTransport):
stderr=subprocess.PIPE,
text=True,
)
+ self._reader_thread = threading.Thread(
+ target=self._read_stdout,
+ name="openjarvis-mcp-stdout",
+ daemon=True,
+ )
+ self._reader_thread.start()
# Nothing else reads stderr. Once the child writes more than the OS
# pipe buffer to it, the child blocks on that write and never gets
- # to answer on stdout, deadlocking send()'s readline() (#750). Drain
- # it continuously on a background thread instead.
+ # to answer on stdout. Drain it continuously on a background thread.
self._stderr_thread = threading.Thread(
target=self._drain_stderr,
args=(self._process,),
+ name="openjarvis-mcp-stderr",
daemon=True,
)
self._stderr_thread.start()
+ def _read_stdout(self) -> None:
+ """Read the subprocess pipe without tying up the request thread.
+
+ A dedicated reader is portable to Windows, where ``select`` cannot
+ wait on an anonymous subprocess pipe. The bounded queue also applies
+ backpressure if a server writes stdout while no request is reading.
+ """
+ proc = self._process
+ stdout = proc.stdout if proc is not None else None
+ if stdout is None:
+ return
+
+ try:
+ while not self._reader_stop.is_set():
+ line = stdout.readline()
+ if not line:
+ break
+ while not self._reader_stop.is_set():
+ try:
+ self._stdout_queue.put(line, timeout=0.1)
+ break
+ except queue.Full:
+ continue
+ finally:
+ while not self._reader_stop.is_set():
+ try:
+ self._stdout_queue.put(self._STDOUT_EOF, timeout=0.1)
+ break
+ except queue.Full:
+ continue
+
+ def _next_stdout_line(self, deadline: float, request_id: Any) -> str:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise RuntimeError(f"Timed out waiting for MCP response id {request_id!r}")
+ try:
+ item = self._stdout_queue.get(timeout=remaining)
+ except queue.Empty as exc:
+ raise RuntimeError(
+ f"Timed out waiting for MCP response id {request_id!r}"
+ ) from exc
+ if item is self._STDOUT_EOF:
+ raise RuntimeError("No response from subprocess")
+ return item
+
def _drain_stderr(self, proc: subprocess.Popen[str]) -> None:
"""Continuously read the child's stderr so its pipe never fills."""
if proc.stderr is None:
@@ -98,19 +173,45 @@ class StdioTransport(MCPTransport):
logger.debug("[%s stderr] %s", self._command[0], line)
def send(self, request: MCPRequest) -> MCPResponse:
- """Write request as JSON line, read response line."""
- proc = self._process
- if proc is None or proc.stdin is None or proc.stdout is None:
- raise RuntimeError("Transport process is not running")
-
- line = request.to_json() + "\n"
- proc.stdin.write(line)
- proc.stdin.flush()
+ """Write request as JSON line, read lines until the matching response.
- response_line = proc.stdout.readline()
- if not response_line:
- raise RuntimeError("No response from subprocess")
- return MCPResponse.from_json(response_line.strip())
+ MCP servers may emit unsolicited notifications or stray/stale
+ replies on stdout before the real response. Skip anything that
+ isn't a well-formed JSON-RPC response carrying this request's id,
+ rather than treating the first line as gospel (#751).
+ """
+ with self._request_lock:
+ proc = self._process
+ if proc is None or proc.stdin is None or proc.stdout is None:
+ raise RuntimeError("Transport process is not running")
+
+ line = request.to_json() + "\n"
+ proc.stdin.write(line)
+ proc.stdin.flush()
+
+ deadline = time.monotonic() + self._response_timeout
+ while True:
+ # Check the wall-clock deadline even when a server continuously
+ # floods stdout, so queued blank/noise/notification lines cannot
+ # keep extending the request forever.
+ response_line = self._next_stdout_line(deadline, request.id)
+ response_line = response_line.strip()
+ if not response_line:
+ continue
+
+ try:
+ parsed = json.loads(response_line)
+ except (json.JSONDecodeError, ValueError):
+ parsed = None
+
+ is_response = (
+ isinstance(parsed, dict)
+ and "id" in parsed
+ and ("result" in parsed or "error" in parsed)
+ and parsed["id"] == request.id
+ )
+ if is_response:
+ return MCPResponse.from_json(response_line)
def send_notification(self, request: MCPRequest) -> None:
"""Send a JSON-RPC notification — write only, never read.
@@ -119,18 +220,36 @@ class StdioTransport(MCPTransport):
to notifications, so the default ``send()`` would block forever
on ``proc.stdout.readline()``.
"""
- proc = self._process
- if proc is None or proc.stdin is None:
- raise RuntimeError("Transport process is not running")
- line = request.to_json() + "\n"
- proc.stdin.write(line)
- proc.stdin.flush()
+ with self._request_lock:
+ proc = self._process
+ if proc is None or proc.stdin is None:
+ raise RuntimeError("Transport process is not running")
+ line = request.to_json() + "\n"
+ proc.stdin.write(line)
+ proc.stdin.flush()
def close(self) -> None:
"""Terminate the subprocess."""
if self._process is not None:
+ self._reader_stop.set()
+ # Wake a request waiting on the queue before stopping the process.
+ try:
+ self._stdout_queue.put_nowait(self._STDOUT_EOF)
+ except queue.Full:
+ try:
+ self._stdout_queue.get_nowait()
+ self._stdout_queue.put_nowait(self._STDOUT_EOF)
+ except (queue.Empty, queue.Full):
+ pass
self._process.terminate()
- self._process.wait(timeout=5)
+ try:
+ self._process.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ self._process.kill()
+ self._process.wait(timeout=5)
+ if self._reader_thread is not None:
+ self._reader_thread.join(timeout=1)
+ self._reader_thread = None
self._process = None
if self._stderr_thread is not None:
self._stderr_thread.join(timeout=5)
diff --git a/tests/mcp/test_transport.py b/tests/mcp/test_transport.py
index 8fb3e129..04a33682 100644
--- a/tests/mcp/test_transport.py
+++ b/tests/mcp/test_transport.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import json
import sys
import textwrap
+import time
from unittest.mock import MagicMock, patch
import pytest
@@ -136,6 +137,291 @@ class TestStdioTransport:
finally:
transport.close()
+ def test_concurrent_requests_remain_correlated(self, tmp_path):
+ """Concurrent callers cannot consume one another's response lines."""
+ import threading
+
+ script = tmp_path / "concurrent_echo_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import json
+ import sys
+ import time
+
+ for line in sys.stdin:
+ request = json.loads(line)
+ time.sleep(0.005)
+ response = {
+ "jsonrpc": "2.0",
+ "id": request["id"],
+ "result": {"method": request["method"]},
+ }
+ sys.stdout.write(json.dumps(response) + "\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)], response_timeout=2.0)
+ barrier = threading.Barrier(12)
+ responses: dict[int, str] = {}
+ errors: list[Exception] = []
+
+ def call(request_id: int) -> None:
+ try:
+ barrier.wait(timeout=2)
+ response = transport.send(
+ MCPRequest(method=f"test/{request_id}", id=request_id)
+ )
+ responses[request_id] = response.result["method"]
+ except Exception as exc: # pragma: no cover - asserted below
+ errors.append(exc)
+
+ workers = [threading.Thread(target=call, args=(index,)) for index in range(12)]
+ try:
+ for worker in workers:
+ worker.start()
+ for worker in workers:
+ worker.join(timeout=5)
+
+ assert not errors
+ assert not any(worker.is_alive() for worker in workers)
+ assert responses == {index: f"test/{index}" for index in range(12)}
+ finally:
+ transport.close()
+
+ def test_skips_unsolicited_notification_before_response(self, tmp_path):
+ """Regression for #751: a server-emitted notification (no ``id``)
+ arriving on stdout before the real response must not be mistaken
+ for that response.
+
+ The old ``send()`` read exactly one line and handed it straight to
+ ``MCPResponse.from_json()``. A notification has neither ``result``
+ nor ``error``, so it parsed as ``MCPResponse(result=None,
+ error=None)`` -- silently wrong instead of erroring -- and the real
+ response was left unread in the pipe, desyncing every later call.
+ """
+ script = tmp_path / "noisy_notify_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import sys
+ import json
+ for line in sys.stdin:
+ line = line.strip()
+ if not line:
+ continue
+ req = json.loads(line)
+ # Unsolicited notification (no "id") before the real reply.
+ notice = {
+ "jsonrpc": "2.0",
+ "method": "notifications/progress",
+ "params": {"progress": 1},
+ }
+ sys.stdout.write(json.dumps(notice) + "\\n")
+ resp = {
+ "jsonrpc": "2.0",
+ "id": req.get("id", 0),
+ "result": {"echo": req.get("method", "")},
+ }
+ sys.stdout.write(json.dumps(resp) + "\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)])
+ try:
+ req = MCPRequest(method="test/echo", id=1)
+ resp = transport.send(req)
+ assert resp.error is None
+ assert resp.result == {"echo": "test/echo"}
+ assert resp.id == 1
+
+ # The stream must be back in sync for the next call too.
+ req2 = MCPRequest(method="test/echo2", id=2)
+ resp2 = transport.send(req2)
+ assert resp2.result == {"echo": "test/echo2"}
+ finally:
+ transport.close()
+
+ def test_skips_response_with_stale_id(self, tmp_path):
+ """A response line whose id doesn't match the outstanding request
+ (e.g. a stale/duplicate reply) must be skipped, not returned."""
+ script = tmp_path / "stale_id_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import sys
+ import json
+ for line in sys.stdin:
+ line = line.strip()
+ if not line:
+ continue
+ req = json.loads(line)
+ stale = {"jsonrpc": "2.0", "id": 999, "result": {"stale": True}}
+ sys.stdout.write(json.dumps(stale) + "\\n")
+ resp = {
+ "jsonrpc": "2.0",
+ "id": req.get("id", 0),
+ "result": {"echo": req.get("method", "")},
+ }
+ sys.stdout.write(json.dumps(resp) + "\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)])
+ try:
+ req = MCPRequest(method="test/echo", id=1)
+ resp = transport.send(req)
+ assert resp.result == {"echo": "test/echo"}
+ assert resp.id == 1
+ finally:
+ transport.close()
+
+ def test_skips_non_json_banner_before_response(self, tmp_path):
+ """Regression for #752: a stray non-JSON line on stdout (e.g. an
+ npm warning banner from a server launched via ``npx -y`` on a cold
+ cache) must be skipped, not treated as a fatal parse error that
+ drops discovery for the whole server.
+ """
+ script = tmp_path / "npm_banner_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import sys
+ import json
+
+ # Simulates an npm warning banner printed before the server
+ # starts speaking JSON-RPC.
+ sys.stdout.write("npm warn deprecated [email protected]: use v2\\n")
+ sys.stdout.write("Fetching latest version...\\n")
+ sys.stdout.flush()
+
+ for line in sys.stdin:
+ line = line.strip()
+ if not line:
+ continue
+ req = json.loads(line)
+ resp = {
+ "jsonrpc": "2.0",
+ "id": req.get("id", 0),
+ "result": {"echo": req.get("method", "")},
+ }
+ sys.stdout.write(json.dumps(resp) + "\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)])
+ try:
+ req = MCPRequest(method="initialize", id=1)
+ resp = transport.send(req)
+ assert resp.error is None
+ assert resp.result == {"echo": "initialize"}
+ finally:
+ transport.close()
+
+ def test_response_deadline_bounds_nonmatching_lines(self, tmp_path):
+ """Noise cannot keep extending the wall-clock response deadline."""
+ script = tmp_path / "never_replies_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import sys
+ import json
+ for line in sys.stdin:
+ line = line.strip()
+ if not line:
+ continue
+ # Never answers with a matching id -- just noise, forever.
+ for _ in range(2000):
+ notice = {"jsonrpc": "2.0", "method": "notifications/progress"}
+ sys.stdout.write(json.dumps(notice) + "\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)], response_timeout=0.25)
+ try:
+ req = MCPRequest(method="test/echo", id=1)
+ started = time.monotonic()
+ with pytest.raises(RuntimeError, match="Timed out waiting"):
+ transport.send(req)
+ assert time.monotonic() - started < 2
+ finally:
+ transport.close()
+
+ def test_response_deadline_bounds_blank_line_flood(self, tmp_path):
+ """Blank lines are ignored, but cannot make send() wait forever."""
+ script = tmp_path / "blank_line_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import sys
+ for line in sys.stdin:
+ while True:
+ sys.stdout.write("\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)], response_timeout=0.25)
+ try:
+ started = time.monotonic()
+ with pytest.raises(RuntimeError, match="Timed out waiting"):
+ transport.send(MCPRequest(method="test/echo", id=1))
+ assert time.monotonic() - started < 2
+ finally:
+ transport.close()
+
+ def test_valid_response_survives_large_notification_burst(self, tmp_path):
+ """Valid protocol traffic is not rejected by an arbitrary line cap."""
+ script = tmp_path / "notification_burst_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import json
+ import sys
+ for line in sys.stdin:
+ request = json.loads(line)
+ for index in range(1200):
+ notification = {
+ "jsonrpc": "2.0",
+ "method": "notifications/progress",
+ "params": {"progress": index},
+ }
+ sys.stdout.write(json.dumps(notification) + "\\n")
+ response = {
+ "jsonrpc": "2.0",
+ "id": request["id"],
+ "result": {"complete": True},
+ }
+ sys.stdout.write(json.dumps(response) + "\\n")
+ sys.stdout.flush()
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)], response_timeout=5)
+ try:
+ response = transport.send(MCPRequest(method="test/echo", id=1))
+ assert response.result == {"complete": True}
+ finally:
+ transport.close()
+
+ def test_response_deadline_bounds_silent_server(self, tmp_path):
+ script = tmp_path / "silent_server.py"
+ script.write_text(
+ textwrap.dedent("""\
+ import sys
+ import time
+ for line in sys.stdin:
+ time.sleep(300)
+ """)
+ )
+
+ transport = StdioTransport([sys.executable, str(script)], response_timeout=0.25)
+ try:
+ started = time.monotonic()
+ with pytest.raises(RuntimeError, match="Timed out waiting"):
+ transport.send(MCPRequest(method="test/echo", id=1))
+ assert time.monotonic() - started < 2
+ finally:
+ transport.close()
+
def test_send_notification_does_not_read_stdout(self, tmp_path):
"""Regression for #339: stdio servers don't reply to notifications.