Personal AI, On Personal Devices
Python
fix(mcp): validate response id in StdioTransport.send() (#798)
havolline194bdb402225b
diff --git a/src/openjarvis/mcp/transport.py b/src/openjarvis/mcp/transport.pyindex bfeb1b88..415e5112 100644--- a/src/openjarvis/mcp/transport.py+++ b/src/openjarvis/mcp/transport.py@@ -2,9 +2,12 @@from __future__ import annotations+import jsonimport logging+import queueimport subprocessimport threading+import timefrom abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, Any, List, Optional@@ -62,10 +65,31 @@ class StdioTransport(MCPTransport):stdin/stdout."""- def __init__(self, command: List[str]) -> None:+ _STDOUT_QUEUE_SIZE = 1024+ _STDOUT_EOF = object()++ def __init__(+ self,+ command: List[str],+ *,+ response_timeout: float = 600.0,+ ) -> None:+ if response_timeout <= 0:+ raise ValueError("response_timeout must be positive")self._command = command+ self._response_timeout = response_timeoutself._process: Optional[subprocess.Popen[str]] = None+ self._stdout_queue: queue.Queue[Any] = queue.Queue(+ maxsize=self._STDOUT_QUEUE_SIZE+ )+ self._reader_stop = threading.Event()+ self._reader_thread: Optional[threading.Thread] = Noneself._stderr_thread: Optional[threading.Thread] = None+ # A single stdout stream cannot safely serve multiple independent+ # readers: one request could consume and discard another request's+ # response. Serialize complete write/read exchanges (and notification+ # writes) so response correlation remains lossless.+ self._request_lock = threading.Lock()self._start()def _start(self) -> None:@@ -77,17 +101,68 @@ class StdioTransport(MCPTransport):stderr=subprocess.PIPE,text=True,)+ self._reader_thread = threading.Thread(+ target=self._read_stdout,+ name="openjarvis-mcp-stdout",+ daemon=True,+ )+ self._reader_thread.start()# Nothing else reads stderr. Once the child writes more than the OS# pipe buffer to it, the child blocks on that write and never gets- # to answer on stdout, deadlocking send()'s readline() (#750). Drain- # it continuously on a background thread instead.+ # to answer on stdout. Drain it continuously on a background thread.self._stderr_thread = threading.Thread(target=self._drain_stderr,args=(self._process,),+ name="openjarvis-mcp-stderr",daemon=True,)self._stderr_thread.start()+ def _read_stdout(self) -> None:+ """Read the subprocess pipe without tying up the request thread.++ A dedicated reader is portable to Windows, where ``select`` cannot+ wait on an anonymous subprocess pipe. The bounded queue also applies+ backpressure if a server writes stdout while no request is reading.+ """+ proc = self._process+ stdout = proc.stdout if proc is not None else None+ if stdout is None:+ return++ try:+ while not self._reader_stop.is_set():+ line = stdout.readline()+ if not line:+ break+ while not self._reader_stop.is_set():+ try:+ self._stdout_queue.put(line, timeout=0.1)+ break+ except queue.Full:+ continue+ finally:+ while not self._reader_stop.is_set():+ try:+ self._stdout_queue.put(self._STDOUT_EOF, timeout=0.1)+ break+ except queue.Full:+ continue++ def _next_stdout_line(self, deadline: float, request_id: Any) -> str:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ raise RuntimeError(f"Timed out waiting for MCP response id {request_id!r}")+ try:+ item = self._stdout_queue.get(timeout=remaining)+ except queue.Empty as exc:+ raise RuntimeError(+ f"Timed out waiting for MCP response id {request_id!r}"+ ) from exc+ if item is self._STDOUT_EOF:+ raise RuntimeError("No response from subprocess")+ return item+def _drain_stderr(self, proc: subprocess.Popen[str]) -> None:"""Continuously read the child's stderr so its pipe never fills."""if proc.stderr is None:@@ -98,19 +173,45 @@ class StdioTransport(MCPTransport):logger.debug("[%s stderr] %s", self._command[0], line)def send(self, request: MCPRequest) -> MCPResponse:- """Write request as JSON line, read response line."""- proc = self._process- if proc is None or proc.stdin is None or proc.stdout is None:- raise RuntimeError("Transport process is not running")-- line = request.to_json() + "\n"- proc.stdin.write(line)- proc.stdin.flush()+ """Write request as JSON line, read lines until the matching response.- response_line = proc.stdout.readline()- if not response_line:- raise RuntimeError("No response from subprocess")- return MCPResponse.from_json(response_line.strip())+ MCP servers may emit unsolicited notifications or stray/stale+ replies on stdout before the real response. Skip anything that+ isn't a well-formed JSON-RPC response carrying this request's id,+ rather than treating the first line as gospel (#751).+ """+ with self._request_lock:+ proc = self._process+ if proc is None or proc.stdin is None or proc.stdout is None:+ raise RuntimeError("Transport process is not running")++ line = request.to_json() + "\n"+ proc.stdin.write(line)+ proc.stdin.flush()++ deadline = time.monotonic() + self._response_timeout+ while True:+ # Check the wall-clock deadline even when a server continuously+ # floods stdout, so queued blank/noise/notification lines cannot+ # keep extending the request forever.+ response_line = self._next_stdout_line(deadline, request.id)+ response_line = response_line.strip()+ if not response_line:+ continue++ try:+ parsed = json.loads(response_line)+ except (json.JSONDecodeError, ValueError):+ parsed = None++ is_response = (+ isinstance(parsed, dict)+ and "id" in parsed+ and ("result" in parsed or "error" in parsed)+ and parsed["id"] == request.id+ )+ if is_response:+ return MCPResponse.from_json(response_line)def send_notification(self, request: MCPRequest) -> None:"""Send a JSON-RPC notification — write only, never read.@@ -119,18 +220,36 @@ class StdioTransport(MCPTransport):to notifications, so the default ``send()`` would block foreveron ``proc.stdout.readline()``."""- proc = self._process- if proc is None or proc.stdin is None:- raise RuntimeError("Transport process is not running")- line = request.to_json() + "\n"- proc.stdin.write(line)- proc.stdin.flush()+ with self._request_lock:+ proc = self._process+ if proc is None or proc.stdin is None:+ raise RuntimeError("Transport process is not running")+ line = request.to_json() + "\n"+ proc.stdin.write(line)+ proc.stdin.flush()def close(self) -> None:"""Terminate the subprocess."""if self._process is not None:+ self._reader_stop.set()+ # Wake a request waiting on the queue before stopping the process.+ try:+ self._stdout_queue.put_nowait(self._STDOUT_EOF)+ except queue.Full:+ try:+ self._stdout_queue.get_nowait()+ self._stdout_queue.put_nowait(self._STDOUT_EOF)+ except (queue.Empty, queue.Full):+ passself._process.terminate()- self._process.wait(timeout=5)+ try:+ self._process.wait(timeout=5)+ except subprocess.TimeoutExpired:+ self._process.kill()+ self._process.wait(timeout=5)+ if self._reader_thread is not None:+ self._reader_thread.join(timeout=1)+ self._reader_thread = Noneself._process = Noneif self._stderr_thread is not None:self._stderr_thread.join(timeout=5)diff --git a/tests/mcp/test_transport.py b/tests/mcp/test_transport.pyindex 8fb3e129..04a33682 100644--- a/tests/mcp/test_transport.py+++ b/tests/mcp/test_transport.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport jsonimport sysimport textwrap+import timefrom unittest.mock import MagicMock, patchimport pytest@@ -136,6 +137,291 @@ class TestStdioTransport:finally:transport.close()+ def test_concurrent_requests_remain_correlated(self, tmp_path):+ """Concurrent callers cannot consume one another's response lines."""+ import threading++ script = tmp_path / "concurrent_echo_server.py"+ script.write_text(+ textwrap.dedent("""\+ import json+ import sys+ import time++ for line in sys.stdin:+ request = json.loads(line)+ time.sleep(0.005)+ response = {+ "jsonrpc": "2.0",+ "id": request["id"],+ "result": {"method": request["method"]},+ }+ sys.stdout.write(json.dumps(response) + "\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)], response_timeout=2.0)+ barrier = threading.Barrier(12)+ responses: dict[int, str] = {}+ errors: list[Exception] = []++ def call(request_id: int) -> None:+ try:+ barrier.wait(timeout=2)+ response = transport.send(+ MCPRequest(method=f"test/{request_id}", id=request_id)+ )+ responses[request_id] = response.result["method"]+ except Exception as exc: # pragma: no cover - asserted below+ errors.append(exc)++ workers = [threading.Thread(target=call, args=(index,)) for index in range(12)]+ try:+ for worker in workers:+ worker.start()+ for worker in workers:+ worker.join(timeout=5)++ assert not errors+ assert not any(worker.is_alive() for worker in workers)+ assert responses == {index: f"test/{index}" for index in range(12)}+ finally:+ transport.close()++ def test_skips_unsolicited_notification_before_response(self, tmp_path):+ """Regression for #751: a server-emitted notification (no ``id``)+ arriving on stdout before the real response must not be mistaken+ for that response.++ The old ``send()`` read exactly one line and handed it straight to+ ``MCPResponse.from_json()``. A notification has neither ``result``+ nor ``error``, so it parsed as ``MCPResponse(result=None,+ error=None)`` -- silently wrong instead of erroring -- and the real+ response was left unread in the pipe, desyncing every later call.+ """+ script = tmp_path / "noisy_notify_server.py"+ script.write_text(+ textwrap.dedent("""\+ import sys+ import json+ for line in sys.stdin:+ line = line.strip()+ if not line:+ continue+ req = json.loads(line)+ # Unsolicited notification (no "id") before the real reply.+ notice = {+ "jsonrpc": "2.0",+ "method": "notifications/progress",+ "params": {"progress": 1},+ }+ sys.stdout.write(json.dumps(notice) + "\\n")+ resp = {+ "jsonrpc": "2.0",+ "id": req.get("id", 0),+ "result": {"echo": req.get("method", "")},+ }+ sys.stdout.write(json.dumps(resp) + "\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)])+ try:+ req = MCPRequest(method="test/echo", id=1)+ resp = transport.send(req)+ assert resp.error is None+ assert resp.result == {"echo": "test/echo"}+ assert resp.id == 1++ # The stream must be back in sync for the next call too.+ req2 = MCPRequest(method="test/echo2", id=2)+ resp2 = transport.send(req2)+ assert resp2.result == {"echo": "test/echo2"}+ finally:+ transport.close()++ def test_skips_response_with_stale_id(self, tmp_path):+ """A response line whose id doesn't match the outstanding request+ (e.g. a stale/duplicate reply) must be skipped, not returned."""+ script = tmp_path / "stale_id_server.py"+ script.write_text(+ textwrap.dedent("""\+ import sys+ import json+ for line in sys.stdin:+ line = line.strip()+ if not line:+ continue+ req = json.loads(line)+ stale = {"jsonrpc": "2.0", "id": 999, "result": {"stale": True}}+ sys.stdout.write(json.dumps(stale) + "\\n")+ resp = {+ "jsonrpc": "2.0",+ "id": req.get("id", 0),+ "result": {"echo": req.get("method", "")},+ }+ sys.stdout.write(json.dumps(resp) + "\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)])+ try:+ req = MCPRequest(method="test/echo", id=1)+ resp = transport.send(req)+ assert resp.result == {"echo": "test/echo"}+ assert resp.id == 1+ finally:+ transport.close()++ def test_skips_non_json_banner_before_response(self, tmp_path):+ """Regression for #752: a stray non-JSON line on stdout (e.g. an+ npm warning banner from a server launched via ``npx -y`` on a cold+ cache) must be skipped, not treated as a fatal parse error that+ drops discovery for the whole server.+ """+ script = tmp_path / "npm_banner_server.py"+ script.write_text(+ textwrap.dedent("""\+ import sys+ import json++ # Simulates an npm warning banner printed before the server+ # starts speaking JSON-RPC.+ sys.stdout.write("npm warn deprecated [email protected]: use v2\\n")+ sys.stdout.write("Fetching latest version...\\n")+ sys.stdout.flush()++ for line in sys.stdin:+ line = line.strip()+ if not line:+ continue+ req = json.loads(line)+ resp = {+ "jsonrpc": "2.0",+ "id": req.get("id", 0),+ "result": {"echo": req.get("method", "")},+ }+ sys.stdout.write(json.dumps(resp) + "\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)])+ try:+ req = MCPRequest(method="initialize", id=1)+ resp = transport.send(req)+ assert resp.error is None+ assert resp.result == {"echo": "initialize"}+ finally:+ transport.close()++ def test_response_deadline_bounds_nonmatching_lines(self, tmp_path):+ """Noise cannot keep extending the wall-clock response deadline."""+ script = tmp_path / "never_replies_server.py"+ script.write_text(+ textwrap.dedent("""\+ import sys+ import json+ for line in sys.stdin:+ line = line.strip()+ if not line:+ continue+ # Never answers with a matching id -- just noise, forever.+ for _ in range(2000):+ notice = {"jsonrpc": "2.0", "method": "notifications/progress"}+ sys.stdout.write(json.dumps(notice) + "\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)], response_timeout=0.25)+ try:+ req = MCPRequest(method="test/echo", id=1)+ started = time.monotonic()+ with pytest.raises(RuntimeError, match="Timed out waiting"):+ transport.send(req)+ assert time.monotonic() - started < 2+ finally:+ transport.close()++ def test_response_deadline_bounds_blank_line_flood(self, tmp_path):+ """Blank lines are ignored, but cannot make send() wait forever."""+ script = tmp_path / "blank_line_server.py"+ script.write_text(+ textwrap.dedent("""\+ import sys+ for line in sys.stdin:+ while True:+ sys.stdout.write("\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)], response_timeout=0.25)+ try:+ started = time.monotonic()+ with pytest.raises(RuntimeError, match="Timed out waiting"):+ transport.send(MCPRequest(method="test/echo", id=1))+ assert time.monotonic() - started < 2+ finally:+ transport.close()++ def test_valid_response_survives_large_notification_burst(self, tmp_path):+ """Valid protocol traffic is not rejected by an arbitrary line cap."""+ script = tmp_path / "notification_burst_server.py"+ script.write_text(+ textwrap.dedent("""\+ import json+ import sys+ for line in sys.stdin:+ request = json.loads(line)+ for index in range(1200):+ notification = {+ "jsonrpc": "2.0",+ "method": "notifications/progress",+ "params": {"progress": index},+ }+ sys.stdout.write(json.dumps(notification) + "\\n")+ response = {+ "jsonrpc": "2.0",+ "id": request["id"],+ "result": {"complete": True},+ }+ sys.stdout.write(json.dumps(response) + "\\n")+ sys.stdout.flush()+ """)+ )++ transport = StdioTransport([sys.executable, str(script)], response_timeout=5)+ try:+ response = transport.send(MCPRequest(method="test/echo", id=1))+ assert response.result == {"complete": True}+ finally:+ transport.close()++ def test_response_deadline_bounds_silent_server(self, tmp_path):+ script = tmp_path / "silent_server.py"+ script.write_text(+ textwrap.dedent("""\+ import sys+ import time+ for line in sys.stdin:+ time.sleep(300)+ """)+ )++ transport = StdioTransport([sys.executable, str(script)], response_timeout=0.25)+ try:+ started = time.monotonic()+ with pytest.raises(RuntimeError, match="Timed out waiting"):+ transport.send(MCPRequest(method="test/echo", id=1))+ assert time.monotonic() - started < 2+ finally:+ transport.close()+def test_send_notification_does_not_read_stdout(self, tmp_path):"""Regression for #339: stdio servers don't reply to notifications.