DSPy: The framework for programming—not prompting—language models
Python
fix(interpreter): desync interpreter on unhandled sandbox rejections (#10190)
Michael I.6334c2ba54b5
diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex a947982cb..57b0ed393 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -38,6 +38,13 @@ LARGE_VAR_THRESHOLD = 100 * 1024 * 1024# JSON-RPC 2.0 Helpers# =============================================================================+# JSON-RPC 2.0 protocol errors+JSONRPC_PROTOCOL_ERRORS = {+ "ParseError": -32700,+ "InvalidRequest": -32600,+ "MethodNotFound": -32601,+}+# Application errors (range: -32000 to -32099)JSONRPC_APP_ERRORS = {"SyntaxError": -32000,@@ -242,6 +249,7 @@ class PythonInterpreter:self.deno_process = Noneself._mounted_files = Falseself._request_id = 0+ self._last_diagnostic: str | None = Noneself._owner_thread: int | None = Noneself._pending_large_vars = {}self._session_ended = False@@ -469,11 +477,12 @@ class PythonInterpreter:if response_line:return response_line+ diagnostic = f" (last sandbox diagnostic: {self._last_diagnostic})" if self._last_diagnostic else ""exit_code = self.deno_process.poll()if exit_code is not None:stderr = self.deno_process.stderr.read() if self.deno_process.stderr else ""- self._raise_terminal_error(f"Deno exited (code {exit_code}) {context}: {stderr}")- self._raise_terminal_error(f"No response {context}")+ self._raise_terminal_error(f"Deno exited (code {exit_code}) {context}: {stderr}{diagnostic}")+ self._raise_terminal_error(f"No response {context}{diagnostic}")def _parse_response_line(self, response_line: str, context: str) -> dict | None:"""Parse a JSON-RPC line, returning None for non-JSON or malformed lines."""@@ -487,14 +496,36 @@ class PythonInterpreter:logger.debug("Skipping malformed JSON during %s: %s", context, response_line[:100])return None+ def _next_request_id(self) -> int:+ self._request_id += 1+ self._last_diagnostic = None+ return self._request_id++ def _handle_out_of_band_message(self, msg: dict, context: str) -> bool:+ """Consume a message that is not a response to any request (a notification+ or an unsolicited id-less error), returning True so the caller keeps+ reading for the real response. Protocol errors with no id are terminal.+ """+ if "method" in msg and "id" not in msg:+ payload, level = msg.get("params"), logging.DEBUG+ elif "error" in msg and msg.get("id") is None:+ payload, level = msg["error"], logging.WARNING+ if isinstance(payload, dict) and payload.get("code") in JSONRPC_PROTOCOL_ERRORS.values():+ self._raise_terminal_error(f"Protocol error {context}: {payload.get('message', msg)}")+ else:+ return False++ self._last_diagnostic = (payload.get("message") if isinstance(payload, dict) else None) or str(msg)+ logger.log(level, "Skipping out-of-band sandbox message %s: %s", context, self._last_diagnostic)+ return True+def _send_request(self, method: str, params: dict, context: str) -> dict:"""Send a JSON-RPC request and return the parsed response.Non-JSON lines (e.g. Pyodide package loading messages) are skipped,up to ``_MAX_SKIP_LINES`` to prevent unbounded blocking."""- self._request_id += 1- request_id = self._request_id+ request_id = self._next_request_id()msg = _jsonrpc_request(method, params, request_id)self._write_message(msg, context)@@ -502,7 +533,7 @@ class PythonInterpreter:while skipped <= self._MAX_SKIP_LINES:response_line = self._read_response_line(context)response = self._parse_response_line(response_line, context)- if response is None:+ if response is None or self._handle_out_of_band_message(response, context):skipped += 1continue@@ -519,7 +550,7 @@ class PythonInterpreter:self._raise_terminal_error(f"Unexpected response {context}: {response}")return response- self._raise_terminal_error(f"Too many non-JSON lines ({skipped}) {context}")+ self._raise_terminal_error(f"Too many skipped lines ({skipped}) {context}")def _health_check(self) -> None:"""Verify the subprocess is alive by executing a simple expression."""@@ -643,8 +674,7 @@ class PythonInterpreter:self._inject_large_var(name, value)# Send the code as JSON-RPC request- self._request_id += 1- execute_request_id = self._request_id+ execute_request_id = self._next_request_id()input_data = _jsonrpc_request("execute", {"code": code}, execute_request_id)self._write_message(input_data, "during execution")@@ -664,6 +694,10 @@ class PythonInterpreter:self._handle_tool_call(msg)continue+ if self._handle_out_of_band_message(msg, "during execution"):+ skipped += 1+ continue+# Handle success responseif "result" in msg:if msg.get("id") != execute_request_id:@@ -681,9 +715,7 @@ class PythonInterpreter:# Handle error responseif "error" in msg:- # Errors with id=null are unsolicited errors (e.g., unhandled async rejections)- # Treat them as errors for the current request- if msg.get("id") is not None and msg.get("id") != execute_request_id:+ if msg.get("id") != execute_request_id:self._raise_terminal_error(f"Response ID mismatch: expected {execute_request_id}, got {msg.get('id')}")@@ -706,7 +738,7 @@ class PythonInterpreter:# Unexpected message format - neither a recognized method nor a responseself._raise_terminal_error(f"Unexpected message format from sandbox: {msg}")- self._raise_terminal_error(f"Too many non-JSON lines ({skipped}) during execution")+ self._raise_terminal_error(f"Too many skipped lines ({skipped}) during execution")@with_callbacksdef start(self) -> None:diff --git a/dspy/primitives/runner.js b/dspy/primitives/runner.jsindex 170c0f956..b4df1f39f 100644--- a/dspy/primitives/runner.js+++ b/dspy/primitives/runner.js@@ -135,7 +135,9 @@ const jsonrpcError = (code, message, id, data = null) => {// These can occur during async Python <-> JS interopglobalThis.addEventListener("unhandledrejection", (event) => {event.preventDefault();- console.log(jsonrpcError(JSONRPC_APP_ERRORS.RuntimeError, `Unhandled async error: ${event.reason?.message || event.reason}`, null));+ console.log(jsonrpcNotification("unhandled_error", {+ message: `Unhandled async error: ${event.reason?.message || event.reason}`,+ }));});const pyodide = await pyodideModule.loadPyodide();diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.pyindex 0bedbbfd6..d8f50c8ac 100644--- a/tests/primitives/test_python_interpreter.py+++ b/tests/primitives/test_python_interpreter.py@@ -1,5 +1,6 @@import asyncioimport dataclasses+import ioimport jsonimport osimport random@@ -1150,3 +1151,96 @@ def test_enable_read_paths_multiple_files(tmp_path):assert contents["test1.txt"] == "Content 1"assert contents["test2.txt"] == "Content 2"assert contents["test3.txt"] == "Content 3"+++def test_system_exit_is_recoverable_and_session_stays_synced(pooled_interpreter):+ """Regression test for #10165: the unhandled Deno rejection accompanying+ SystemExit must not be consumed as the response, desyncing later requests."""+ interpreter = pooled_interpreter+ with pytest.raises(CodeExecutionError, match="SystemExit"):+ interpreter.execute("import sys\nsys.exit(0)")++ assert interpreter.execute("print('still alive')") == "still alive\n"+++def test_keyboard_interrupt_is_recoverable_and_session_stays_synced(pooled_interpreter):+ """KeyboardInterrupt takes the same asyncio re-raise path as SystemExit (#10165)."""+ interpreter = pooled_interpreter+ with pytest.raises(CodeExecutionError, match="KeyboardInterrupt"):+ interpreter.execute("raise KeyboardInterrupt('stop')")++ assert interpreter.execute("print('still alive')") == "still alive\n"+++def test_base_exception_subclasses_are_recoverable_and_session_stays_synced(pooled_interpreter):+ """Subclasses of SystemExit/KeyboardInterrupt take the same re-raise path (#10165)."""+ interpreter = pooled_interpreter+ for code, error_type in [+ ("class ExitSignal(SystemExit): pass\nraise ExitSignal(0)", "ExitSignal"),+ ("class InterruptSignal(KeyboardInterrupt): pass\nraise InterruptSignal()", "InterruptSignal"),+ ]:+ with pytest.raises(CodeExecutionError, match=error_type):+ interpreter.execute(code)+ assert interpreter.execute("print('still alive')") == "still alive\n"+++def test_out_of_band_messages_are_skipped_not_consumed_as_responses():+ """Notifications and unsolicited id-less errors are diagnostics, not responses (#10165)."""+ interpreter = PythonInterpreter()++ notification = {"jsonrpc": "2.0", "method": "unhandled_error", "params": {"message": "boom"}}+ assert interpreter._handle_out_of_band_message(notification, "during test")+ assert interpreter._last_diagnostic == "boom"++ unsolicited_error = {"jsonrpc": "2.0", "error": {"code": -32007, "message": "crash"}, "id": None}+ assert interpreter._handle_out_of_band_message(unsolicited_error, "during test")+ assert interpreter._last_diagnostic == "crash"++ # Real responses (success or error, with an id) must not be consumed.+ result = {"jsonrpc": "2.0", "result": {"output": "2\n"}, "id": 1}+ assert not interpreter._handle_out_of_band_message(result, "during test")+ error_response = {"jsonrpc": "2.0", "error": {"code": -32007, "message": "boom"}, "id": 1}+ assert not interpreter._handle_out_of_band_message(error_response, "during test")+++def test_id_less_protocol_errors_are_terminal():+ """ParseError/InvalidRequest mean the sandbox never read the request, so no+ response will follow; waiting for one would block forever (#10165)."""+ interpreter = PythonInterpreter()+ parse_error = {"jsonrpc": "2.0", "error": {"code": -32700, "message": "Invalid JSON input"}, "id": None}+ with pytest.raises(CodeInterpreterError, match="Protocol error"):+ interpreter._handle_out_of_band_message(parse_error, "during test")+++def test_unsolicited_error_line_is_not_consumed_as_the_response():+ """#10165: an id-less async error arriving ahead of the real response (the+ wire trace an unhandled rejection produces) must not be mistaken for it."""++ class FakeDeno:+ def __init__(self, lines):+ self.stdin = io.StringIO()+ self.stdout = io.StringIO("".join(line + "\n" for line in lines))+ self.stderr = io.StringIO()++ def poll(self):+ return None++ interpreter = PythonInterpreter()+ interpreter.deno_process = FakeDeno([+ json.dumps({"jsonrpc": "2.0", "error": {"code": -32007, "message": "Unhandled async error: PythonError"}, "id": None}),+ json.dumps({"jsonrpc": "2.0", "result": {"output": "ok\n"}, "id": 1}),+ ])+ assert interpreter.execute("print('ok')") == "ok\n"+++def test_base_exceptions_do_not_desync_interpreter():+ with PythonInterpreter() as interpreter:+ for code, error_type in [+ ("import sys\nsys.exit(0)", "SystemExit"),+ ("raise KeyboardInterrupt()", "KeyboardInterrupt"),+ ("class ExitSignal(SystemExit): pass\nraise ExitSignal(0)", "ExitSignal"),+ ("class InterruptSignal(KeyboardInterrupt): pass\nraise InterruptSignal()", "InterruptSignal"),+ ]:+ with pytest.raises(CodeExecutionError, match=error_type):+ interpreter.execute(code)+ assert interpreter.execute("print('still alive')") == "still alive\n"