DSPy: The framework for programming—not prompting—language models
Python
fix(RLM): large variable injection (#9233)
Isaac Miller815f92a0b602
diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex 830a59155..12f1d6169 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -23,6 +23,10 @@ __all__ = ["PythonInterpreter", "FinalOutput", "CodeInterpreterError"]logger = logging.getLogger(__name__)+# Pyodide's FFI crashes at exactly 128MB (134,217,728 bytes). Use filesystem+# injection for strings above 100MB to stay safely below this limit.+LARGE_VAR_THRESHOLD = 100 * 1024 * 1024+# =============================================================================# JSON-RPC 2.0 Helpers# =============================================================================@@ -179,6 +183,7 @@ class PythonInterpreter:self._mounted_files = Falseself._request_id = 0self._owner_thread: int | None = None+ self._pending_large_vars = {}def _check_thread_ownership(self) -> None:"""Ensure this interpreter is only used from a single thread."""@@ -235,19 +240,7 @@ class PythonInterpreter:else:raise FileNotFoundError(f"Cannot mount non-existent file: {path}")virtual_path = f"/sandbox/{os.path.basename(path)}"- self._request_id += 1- request_id = self._request_id- mount_msg = _jsonrpc_request("mount_file", {"host_path": str(path), "virtual_path": virtual_path}, request_id)- self.deno_process.stdin.write(mount_msg + "\n")- self.deno_process.stdin.flush()- response_line = self.deno_process.stdout.readline().strip()- if not response_line:- raise CodeInterpreterError(f"No response when mounting file: {path}")- response = json.loads(response_line)- if response.get("id") != request_id:- raise CodeInterpreterError(f"Response ID mismatch when mounting: expected {request_id}, got {response.get('id')}")- if "error" in response:- raise CodeInterpreterError(f"Failed to mount {path}: {response['error'].get('message', 'Unknown error')}")+ self._send_request("mount_file", {"host_path": str(path), "virtual_path": virtual_path}, f"mounting {path}")self._mounted_files = Truedef _sync_files(self):@@ -300,17 +293,7 @@ class PythonInterpreter:self._tools_registered = Truereturn- self._request_id += 1- request_id = self._request_id- register_msg = _jsonrpc_request("register", params, request_id)- self.deno_process.stdin.write(register_msg + "\n")- self.deno_process.stdin.flush()- response_line = self.deno_process.stdout.readline().strip()- if not response_line:- raise CodeInterpreterError("No response when registering tools/outputs")- response = json.loads(response_line)- if "result" not in response or response.get("id") != request_id:- raise CodeInterpreterError(f"Unexpected response when registering: {response_line}")+ self._send_request("register", params, "registering tools/outputs")self._tools_registered = Truedef _handle_tool_call(self, request: dict) -> None:@@ -362,26 +345,74 @@ class PythonInterpreter:raise CodeInterpreterError(install_instructions) from eself._health_check()- def _health_check(self) -> None:- """Verify the subprocess is alive by executing a simple expression."""+ def _send_request(self, method: str, params: dict, context: str) -> dict:+ """Send a JSON-RPC request and return the parsed response."""self._request_id += 1request_id = self._request_id- ping_msg = _jsonrpc_request("execute", {"code": "print(1+1)"}, request_id)- self.deno_process.stdin.write(ping_msg + "\n")+ msg = _jsonrpc_request(method, params, request_id)+ self.deno_process.stdin.write(msg + "\n")self.deno_process.stdin.flush()+response_line = self.deno_process.stdout.readline().strip()if not response_line:- raise CodeInterpreterError("Subprocess not responding")+ exit_code = self.deno_process.poll()+ if exit_code is not None:+ stderr = self.deno_process.stderr.read() if self.deno_process.stderr else ""+ raise CodeInterpreterError(f"Deno exited (code {exit_code}) {context}: {stderr}")+ raise CodeInterpreterError(f"No response {context}")+response = json.loads(response_line)+ if response.get("id") != request_id:+ raise CodeInterpreterError(f"Response ID mismatch {context}: expected {request_id}, got {response.get('id')}")+ if "error" in response:+ raise CodeInterpreterError(f"Error {context}: {response['error'].get('message', 'Unknown error')}")+ return response++ def _health_check(self) -> None:+ """Verify the subprocess is alive by executing a simple expression."""+ response = self._send_request("execute", {"code": "print(1+1)"}, "during health check")if response.get("result", {}).get("output", "").strip() != "2":- raise CodeInterpreterError(f"Unexpected ping response: {response_line}")+ raise CodeInterpreterError(f"Unexpected ping response: {response}")++ def _to_json_compatible(self, value: Any) -> Any:+ """Recursively convert Python values to JSON-compatible types."""+ if value is None or isinstance(value, (str, int, float, bool)):+ return value+ elif isinstance(value, dict):+ return {k: self._to_json_compatible(v) for k, v in value.items()}+ elif isinstance(value, (list, tuple)):+ return [self._to_json_compatible(v) for v in value]+ elif isinstance(value, set):+ try:+ return sorted(self._to_json_compatible(v) for v in value)+ except TypeError:+ return [self._to_json_compatible(v) for v in value]+ else:+ raise CodeInterpreterError(f"Unsupported value type: {type(value).__name__}")def _inject_variables(self, code: str, variables: dict[str, Any]) -> str:"""Insert Python assignments for each variable at the top of the code."""for key in variables:- if not key.isidentifier() or keyword.iskeyword(key):+ if not key.isidentifier() or keyword.iskeyword(key) or key == "json":raise CodeInterpreterError(f"Invalid variable name: '{key}'")- assignments = [f"{k} = {self._serialize_value(v)}" for k, v in variables.items()]++ large_vars = {}+ small_assignments = []+ for k, v in variables.items():+ serialized = self._serialize_value(v)+ if len(serialized) > LARGE_VAR_THRESHOLD:+ large_vars[k] = json.dumps(self._to_json_compatible(v))+ else:+ small_assignments.append(f"{k} = {serialized}")++ self._pending_large_vars = large_vars++ if large_vars:+ large_assignments = [f"{k} = json.loads(open('/tmp/dspy_vars/{k}.json').read())" for k in large_vars]+ assignments = ["import json"] + small_assignments + large_assignments+ else:+ assignments = small_assignments+return "\n".join(assignments) + "\n" + code if assignments else codedef _serialize_value(self, value: Any) -> str:@@ -394,18 +425,15 @@ class PythonInterpreter:return str(value)elif isinstance(value, (int, float)):return str(value)- elif isinstance(value, (list, dict)):- return json.dumps(value)- elif isinstance(value, tuple):- return json.dumps(list(value))- elif isinstance(value, set):- try:- return json.dumps(sorted(value))- except TypeError:- return json.dumps(list(value))+ elif isinstance(value, (list, dict, tuple, set)):+ return json.dumps(self._to_json_compatible(value))else:raise CodeInterpreterError(f"Unsupported value type: {type(value).__name__}")+ def _inject_large_var(self, name: str, value: str) -> None:+ """Inject a large variable via the virtual filesystem."""+ self._send_request("inject_var", {"name": name, "value": value}, f"injecting variable '{name}'")+def execute(self,code: str,@@ -418,6 +446,9 @@ class PythonInterpreter:self._mount_files()self._register_tools()+ for name, value in self._pending_large_vars.items():+ self._inject_large_var(name, value)+# Send the code as JSON-RPC requestself._request_id += 1execute_request_id = self._request_id@@ -432,6 +463,8 @@ class PythonInterpreter:self._ensure_deno_process()self._mount_files()self._register_tools()+ for name, value in self._pending_large_vars.items():+ self._inject_large_var(name, value)self.deno_process.stdin.write(input_data + "\n")self.deno_process.stdin.flush()diff --git a/dspy/primitives/runner.js b/dspy/primitives/runner.jsindex 7fc79635e..1c1599b99 100644--- a/dspy/primitives/runner.js+++ b/dspy/primitives/runner.js@@ -306,6 +306,19 @@ while (true) {continue;}+ if (method === "inject_var") {+ const { name, value } = params;+ try {+ try { pyodide.FS.mkdir('/tmp'); } catch (e) { /* exists */ }+ try { pyodide.FS.mkdir('/tmp/dspy_vars'); } catch (e) { /* exists */ }+ pyodide.FS.writeFile(`/tmp/dspy_vars/${name}.json`, new TextEncoder().encode(value));+ console.log(jsonrpcResult({ injected: name }, requestId));+ } catch (e) {+ console.log(jsonrpcError(JSONRPC_APP_ERRORS.RuntimeError, `Failed to inject var: ${e.message}`, requestId));+ }+ continue;+ }+if (method === "execute") {const code = params.code || "";let setupCompleted = false; // Track if PYTHON_SETUP_CODE ran successfullydiff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.pyindex 40b4d1b6d..088e5ab2d 100644--- a/tests/primitives/test_python_interpreter.py+++ b/tests/primitives/test_python_interpreter.py@@ -406,3 +406,131 @@ def test_extract_parameters_complex_types():# Complex types like Union are not included in type annotationassert params[0] == {"name": "items", "default": None}assert params[1] == {"name": "data", "default": None}+++# =============================================================================+# Large Variable Injection Tests+# =============================================================================++def test_large_variable_injection():+ """Test that large strings are injected via filesystem to avoid Pyodide's FFI size limit."""+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD++ # Create a string just over the threshold+ large_data = "x" * (LARGE_VAR_THRESHOLD + 1024)++ with PythonInterpreter() as interpreter:+ result = interpreter.execute("len(data)", variables={"data": large_data})+ assert result == len(large_data), "Large variable should be correctly injected and accessible"+++def test_large_variable_content_integrity():+ """Test that large variable content is preserved exactly through filesystem injection."""+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD++ # Create a string with recognizable pattern just over threshold+ pattern = "ABCDEFGHIJ" * 100+ large_data = pattern * ((LARGE_VAR_THRESHOLD // len(pattern)) + 1)++ with PythonInterpreter() as interpreter:+ # Check first and last parts to verify content integrity+ code = """+first_100 = data[:100]+last_100 = data[-100:]+(first_100, last_100)+"""+ result = interpreter.execute(code, variables={"data": large_data})+ assert result[0] == large_data[:100], "First 100 chars should match"+ assert result[1] == large_data[-100:], "Last 100 chars should match"+++def test_mixed_small_and_large_variables():+ """Test that small and large variables can be used together."""+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD++ small_var = "hello"+ large_var = "x" * (LARGE_VAR_THRESHOLD + 1024)++ with PythonInterpreter() as interpreter:+ code = "f'{small} has {len(small)} chars, large has {len(large)} chars'"+ result = interpreter.execute(code, variables={"small": small_var, "large": large_var})+ expected = f"{small_var} has {len(small_var)} chars, large has {len(large_var)} chars"+ assert result == expected, "Both small and large variables should work together"+++def test_multiple_large_variables():+ """Test that multiple large variables can be injected."""+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD++ large_a = "a" * (LARGE_VAR_THRESHOLD + 100)+ large_b = "b" * (LARGE_VAR_THRESHOLD + 200)++ with PythonInterpreter() as interpreter:+ code = "(len(var_a), len(var_b), var_a[0], var_b[0])"+ result = interpreter.execute(code, variables={"var_a": large_a, "var_b": large_b})+ assert result == [len(large_a), len(large_b), "a", "b"], "Multiple large variables should work"+++def test_large_list_variable():+ """Test that large list variables are injected via filesystem and JSON parsed."""+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD++ # Each element "x" serializes to ~3 chars, so divide threshold by 3+ num_elements = LARGE_VAR_THRESHOLD // 3+ large_list = ["x"] * num_elements++ with PythonInterpreter() as interpreter:+ code = "(len(data), data[0], data[-1], type(data).__name__)"+ result = interpreter.execute(code, variables={"data": large_list})+ assert result == [num_elements, "x", "x", "list"]+++def test_nested_sets_and_tuples():+ """Test that nested structures with sets and tuples are converted to JSON-compatible types."""+ complex_data = {+ "tags": {1, 2, 3},+ "coords": (10, 20),+ "nested": [{"inner_set": {"a", "b"}}]+ }++ with PythonInterpreter() as interpreter:+ result = interpreter.execute("data", variables={"data": complex_data})+ # Sets become sorted lists, tuples become lists+ assert result["tags"] == [1, 2, 3]+ assert result["coords"] == [10, 20]+ assert result["nested"][0]["inner_set"] == ["a", "b"]+++def test_small_variable_not_using_filesystem():+ """Test that small variables are embedded in code, not using filesystem."""+ small_var = "small string"++ interpreter = PythonInterpreter()+ interpreter._pending_large_vars = {} # Initialize+ interpreter._inject_variables("print(x)", {"x": small_var})++ assert interpreter._pending_large_vars == {}, "Small variables should not be in _pending_large_vars"+++def test_large_variable_threshold_boundary():+ """Test behavior at exactly the threshold boundary.++ The threshold applies to the serialized size, not the original value.+ For strings, serialization adds 2 bytes (quotes).+ """+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD++ # Serialized size at threshold - should use embedded (not filesystem)+ # Account for 2 bytes of quotes added by repr()+ at_threshold = "x" * (LARGE_VAR_THRESHOLD - 2)++ interpreter = PythonInterpreter()+ interpreter._pending_large_vars = {}+ interpreter._inject_variables("print(x)", {"x": at_threshold})+ assert interpreter._pending_large_vars == {}, "Serialized size at threshold should be embedded"++ # Serialized size over threshold - should use filesystem+ over_threshold = "x" * (LARGE_VAR_THRESHOLD - 1)+ interpreter._pending_large_vars = {}+ interpreter._inject_variables("print(x)", {"x": over_threshold})+ assert "x" in interpreter._pending_large_vars, "Serialized size over threshold should use filesystem"