Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(RLM): large variable injection (#9233)

Isaac Miller815f92a0b602
diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.py
index 830a59155..12f1d6169 100644
--- a/dspy/primitives/python_interpreter.py
+++ b/dspy/primitives/python_interpreter.py
@@ -23,6 +23,10 @@ __all__ = ["PythonInterpreter", "FinalOutput", "CodeInterpreterError"]
logger = logging.getLogger(__name__)
+# Pyodide's FFI crashes at exactly 128MB (134,217,728 bytes). Use filesystem
+# injection for strings above 100MB to stay safely below this limit.
+LARGE_VAR_THRESHOLD = 100 * 1024 * 1024
+
# =============================================================================
# JSON-RPC 2.0 Helpers
# =============================================================================
@@ -179,6 +183,7 @@ class PythonInterpreter:
self._mounted_files = False
self._request_id = 0
self._owner_thread: int | None = None
+ self._pending_large_vars = {}
def _check_thread_ownership(self) -> None:
"""Ensure this interpreter is only used from a single thread."""
@@ -235,19 +240,7 @@ class PythonInterpreter:
else:
raise FileNotFoundError(f"Cannot mount non-existent file: {path}")
virtual_path = f"/sandbox/{os.path.basename(path)}"
- self._request_id += 1
- request_id = self._request_id
- mount_msg = _jsonrpc_request("mount_file", {"host_path": str(path), "virtual_path": virtual_path}, request_id)
- self.deno_process.stdin.write(mount_msg + "\n")
- self.deno_process.stdin.flush()
- response_line = self.deno_process.stdout.readline().strip()
- if not response_line:
- raise CodeInterpreterError(f"No response when mounting file: {path}")
- response = json.loads(response_line)
- if response.get("id") != request_id:
- raise CodeInterpreterError(f"Response ID mismatch when mounting: expected {request_id}, got {response.get('id')}")
- if "error" in response:
- raise CodeInterpreterError(f"Failed to mount {path}: {response['error'].get('message', 'Unknown error')}")
+ self._send_request("mount_file", {"host_path": str(path), "virtual_path": virtual_path}, f"mounting {path}")
self._mounted_files = True
def _sync_files(self):
@@ -300,17 +293,7 @@ class PythonInterpreter:
self._tools_registered = True
return
- self._request_id += 1
- request_id = self._request_id
- register_msg = _jsonrpc_request("register", params, request_id)
- self.deno_process.stdin.write(register_msg + "\n")
- self.deno_process.stdin.flush()
- response_line = self.deno_process.stdout.readline().strip()
- if not response_line:
- raise CodeInterpreterError("No response when registering tools/outputs")
- response = json.loads(response_line)
- if "result" not in response or response.get("id") != request_id:
- raise CodeInterpreterError(f"Unexpected response when registering: {response_line}")
+ self._send_request("register", params, "registering tools/outputs")
self._tools_registered = True
def _handle_tool_call(self, request: dict) -> None:
@@ -362,26 +345,74 @@ class PythonInterpreter:
raise CodeInterpreterError(install_instructions) from e
self._health_check()
- def _health_check(self) -> None:
- """Verify the subprocess is alive by executing a simple expression."""
+ def _send_request(self, method: str, params: dict, context: str) -> dict:
+ """Send a JSON-RPC request and return the parsed response."""
self._request_id += 1
request_id = self._request_id
- ping_msg = _jsonrpc_request("execute", {"code": "print(1+1)"}, request_id)
- self.deno_process.stdin.write(ping_msg + "\n")
+ msg = _jsonrpc_request(method, params, request_id)
+ self.deno_process.stdin.write(msg + "\n")
self.deno_process.stdin.flush()
+
response_line = self.deno_process.stdout.readline().strip()
if not response_line:
- raise CodeInterpreterError("Subprocess not responding")
+ exit_code = self.deno_process.poll()
+ if exit_code is not None:
+ stderr = self.deno_process.stderr.read() if self.deno_process.stderr else ""
+ raise CodeInterpreterError(f"Deno exited (code {exit_code}) {context}: {stderr}")
+ raise CodeInterpreterError(f"No response {context}")
+
response = json.loads(response_line)
+ if response.get("id") != request_id:
+ raise CodeInterpreterError(f"Response ID mismatch {context}: expected {request_id}, got {response.get('id')}")
+ if "error" in response:
+ raise CodeInterpreterError(f"Error {context}: {response['error'].get('message', 'Unknown error')}")
+ return response
+
+ def _health_check(self) -> None:
+ """Verify the subprocess is alive by executing a simple expression."""
+ response = self._send_request("execute", {"code": "print(1+1)"}, "during health check")
if response.get("result", {}).get("output", "").strip() != "2":
- raise CodeInterpreterError(f"Unexpected ping response: {response_line}")
+ raise CodeInterpreterError(f"Unexpected ping response: {response}")
+
+ def _to_json_compatible(self, value: Any) -> Any:
+ """Recursively convert Python values to JSON-compatible types."""
+ if value is None or isinstance(value, (str, int, float, bool)):
+ return value
+ elif isinstance(value, dict):
+ return {k: self._to_json_compatible(v) for k, v in value.items()}
+ elif isinstance(value, (list, tuple)):
+ return [self._to_json_compatible(v) for v in value]
+ elif isinstance(value, set):
+ try:
+ return sorted(self._to_json_compatible(v) for v in value)
+ except TypeError:
+ return [self._to_json_compatible(v) for v in value]
+ else:
+ raise CodeInterpreterError(f"Unsupported value type: {type(value).__name__}")
def _inject_variables(self, code: str, variables: dict[str, Any]) -> str:
"""Insert Python assignments for each variable at the top of the code."""
for key in variables:
- if not key.isidentifier() or keyword.iskeyword(key):
+ if not key.isidentifier() or keyword.iskeyword(key) or key == "json":
raise CodeInterpreterError(f"Invalid variable name: '{key}'")
- assignments = [f"{k} = {self._serialize_value(v)}" for k, v in variables.items()]
+
+ large_vars = {}
+ small_assignments = []
+ for k, v in variables.items():
+ serialized = self._serialize_value(v)
+ if len(serialized) > LARGE_VAR_THRESHOLD:
+ large_vars[k] = json.dumps(self._to_json_compatible(v))
+ else:
+ small_assignments.append(f"{k} = {serialized}")
+
+ self._pending_large_vars = large_vars
+
+ if large_vars:
+ large_assignments = [f"{k} = json.loads(open('/tmp/dspy_vars/{k}.json').read())" for k in large_vars]
+ assignments = ["import json"] + small_assignments + large_assignments
+ else:
+ assignments = small_assignments
+
return "\n".join(assignments) + "\n" + code if assignments else code
def _serialize_value(self, value: Any) -> str:
@@ -394,18 +425,15 @@ class PythonInterpreter:
return str(value)
elif isinstance(value, (int, float)):
return str(value)
- elif isinstance(value, (list, dict)):
- return json.dumps(value)
- elif isinstance(value, tuple):
- return json.dumps(list(value))
- elif isinstance(value, set):
- try:
- return json.dumps(sorted(value))
- except TypeError:
- return json.dumps(list(value))
+ elif isinstance(value, (list, dict, tuple, set)):
+ return json.dumps(self._to_json_compatible(value))
else:
raise CodeInterpreterError(f"Unsupported value type: {type(value).__name__}")
+ def _inject_large_var(self, name: str, value: str) -> None:
+ """Inject a large variable via the virtual filesystem."""
+ self._send_request("inject_var", {"name": name, "value": value}, f"injecting variable '{name}'")
+
def execute(
self,
code: str,
@@ -418,6 +446,9 @@ class PythonInterpreter:
self._mount_files()
self._register_tools()
+ for name, value in self._pending_large_vars.items():
+ self._inject_large_var(name, value)
+
# Send the code as JSON-RPC request
self._request_id += 1
execute_request_id = self._request_id
@@ -432,6 +463,8 @@ class PythonInterpreter:
self._ensure_deno_process()
self._mount_files()
self._register_tools()
+ for name, value in self._pending_large_vars.items():
+ self._inject_large_var(name, value)
self.deno_process.stdin.write(input_data + "\n")
self.deno_process.stdin.flush()
diff --git a/dspy/primitives/runner.js b/dspy/primitives/runner.js
index 7fc79635e..1c1599b99 100644
--- a/dspy/primitives/runner.js
+++ b/dspy/primitives/runner.js
@@ -306,6 +306,19 @@ while (true) {
continue;
}
+ if (method === "inject_var") {
+ const { name, value } = params;
+ try {
+ try { pyodide.FS.mkdir('/tmp'); } catch (e) { /* exists */ }
+ try { pyodide.FS.mkdir('/tmp/dspy_vars'); } catch (e) { /* exists */ }
+ pyodide.FS.writeFile(`/tmp/dspy_vars/${name}.json`, new TextEncoder().encode(value));
+ console.log(jsonrpcResult({ injected: name }, requestId));
+ } catch (e) {
+ console.log(jsonrpcError(JSONRPC_APP_ERRORS.RuntimeError, `Failed to inject var: ${e.message}`, requestId));
+ }
+ continue;
+ }
+
if (method === "execute") {
const code = params.code || "";
let setupCompleted = false; // Track if PYTHON_SETUP_CODE ran successfully
diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.py
index 40b4d1b6d..088e5ab2d 100644
--- a/tests/primitives/test_python_interpreter.py
+++ b/tests/primitives/test_python_interpreter.py
@@ -406,3 +406,131 @@ def test_extract_parameters_complex_types():
# Complex types like Union are not included in type annotation
assert params[0] == {"name": "items", "default": None}
assert params[1] == {"name": "data", "default": None}
+
+
+# =============================================================================
+# Large Variable Injection Tests
+# =============================================================================
+
+def test_large_variable_injection():
+ """Test that large strings are injected via filesystem to avoid Pyodide's FFI size limit."""
+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD
+
+ # Create a string just over the threshold
+ large_data = "x" * (LARGE_VAR_THRESHOLD + 1024)
+
+ with PythonInterpreter() as interpreter:
+ result = interpreter.execute("len(data)", variables={"data": large_data})
+ assert result == len(large_data), "Large variable should be correctly injected and accessible"
+
+
+def test_large_variable_content_integrity():
+ """Test that large variable content is preserved exactly through filesystem injection."""
+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD
+
+ # Create a string with recognizable pattern just over threshold
+ pattern = "ABCDEFGHIJ" * 100
+ large_data = pattern * ((LARGE_VAR_THRESHOLD // len(pattern)) + 1)
+
+ with PythonInterpreter() as interpreter:
+ # Check first and last parts to verify content integrity
+ code = """
+first_100 = data[:100]
+last_100 = data[-100:]
+(first_100, last_100)
+"""
+ result = interpreter.execute(code, variables={"data": large_data})
+ assert result[0] == large_data[:100], "First 100 chars should match"
+ assert result[1] == large_data[-100:], "Last 100 chars should match"
+
+
+def test_mixed_small_and_large_variables():
+ """Test that small and large variables can be used together."""
+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD
+
+ small_var = "hello"
+ large_var = "x" * (LARGE_VAR_THRESHOLD + 1024)
+
+ with PythonInterpreter() as interpreter:
+ code = "f'{small} has {len(small)} chars, large has {len(large)} chars'"
+ result = interpreter.execute(code, variables={"small": small_var, "large": large_var})
+ expected = f"{small_var} has {len(small_var)} chars, large has {len(large_var)} chars"
+ assert result == expected, "Both small and large variables should work together"
+
+
+def test_multiple_large_variables():
+ """Test that multiple large variables can be injected."""
+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD
+
+ large_a = "a" * (LARGE_VAR_THRESHOLD + 100)
+ large_b = "b" * (LARGE_VAR_THRESHOLD + 200)
+
+ with PythonInterpreter() as interpreter:
+ code = "(len(var_a), len(var_b), var_a[0], var_b[0])"
+ result = interpreter.execute(code, variables={"var_a": large_a, "var_b": large_b})
+ assert result == [len(large_a), len(large_b), "a", "b"], "Multiple large variables should work"
+
+
+def test_large_list_variable():
+ """Test that large list variables are injected via filesystem and JSON parsed."""
+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD
+
+ # Each element "x" serializes to ~3 chars, so divide threshold by 3
+ num_elements = LARGE_VAR_THRESHOLD // 3
+ large_list = ["x"] * num_elements
+
+ with PythonInterpreter() as interpreter:
+ code = "(len(data), data[0], data[-1], type(data).__name__)"
+ result = interpreter.execute(code, variables={"data": large_list})
+ assert result == [num_elements, "x", "x", "list"]
+
+
+def test_nested_sets_and_tuples():
+ """Test that nested structures with sets and tuples are converted to JSON-compatible types."""
+ complex_data = {
+ "tags": {1, 2, 3},
+ "coords": (10, 20),
+ "nested": [{"inner_set": {"a", "b"}}]
+ }
+
+ with PythonInterpreter() as interpreter:
+ result = interpreter.execute("data", variables={"data": complex_data})
+ # Sets become sorted lists, tuples become lists
+ assert result["tags"] == [1, 2, 3]
+ assert result["coords"] == [10, 20]
+ assert result["nested"][0]["inner_set"] == ["a", "b"]
+
+
+def test_small_variable_not_using_filesystem():
+ """Test that small variables are embedded in code, not using filesystem."""
+ small_var = "small string"
+
+ interpreter = PythonInterpreter()
+ interpreter._pending_large_vars = {} # Initialize
+ interpreter._inject_variables("print(x)", {"x": small_var})
+
+ assert interpreter._pending_large_vars == {}, "Small variables should not be in _pending_large_vars"
+
+
+def test_large_variable_threshold_boundary():
+ """Test behavior at exactly the threshold boundary.
+
+ The threshold applies to the serialized size, not the original value.
+ For strings, serialization adds 2 bytes (quotes).
+ """
+ from dspy.primitives.python_interpreter import LARGE_VAR_THRESHOLD
+
+ # Serialized size at threshold - should use embedded (not filesystem)
+ # Account for 2 bytes of quotes added by repr()
+ at_threshold = "x" * (LARGE_VAR_THRESHOLD - 2)
+
+ interpreter = PythonInterpreter()
+ interpreter._pending_large_vars = {}
+ interpreter._inject_variables("print(x)", {"x": at_threshold})
+ assert interpreter._pending_large_vars == {}, "Serialized size at threshold should be embedded"
+
+ # Serialized size over threshold - should use filesystem
+ over_threshold = "x" * (LARGE_VAR_THRESHOLD - 1)
+ interpreter._pending_large_vars = {}
+ interpreter._inject_variables("print(x)", {"x": over_threshold})
+ assert "x" in interpreter._pending_large_vars, "Serialized size over threshold should use filesystem"