DSPy: The framework for programming—not prompting—language models
Python
refactor(rlm): enhance code fence parsing and REPL entry formatting (#9309)
Isaac Miller3109c61fa4e7
diff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.pyindex 66c011d23..48f55f79e 100644--- a/dspy/predict/rlm.py+++ b/dspy/predict/rlm.py@@ -11,7 +11,6 @@ Reference: "Recursive Language Models" (Zhang, Kraska, Khattab, 2025)from __future__ import annotationsimport logging-import reimport threadingfrom concurrent.futures import ThreadPoolExecutor, as_completedfrom contextlib import contextmanager@@ -62,16 +61,41 @@ IMPORTANT: This is ITERATIVE. Each code block you write will execute, you'll seeYou have max {max_llm_calls} sub-LLM calls. When done, call SUBMIT() with your output."""-# Pattern to match markdown code fences: ```python\n...\n``` or ```\n...\n```-_CODE_FENCE_PATTERN = re.compile(r"^```(?:python|py)?\s*\n(.*)\n```\s*$", re.DOTALL)+_PYTHON_FENCE_LANGS = {"python", "py", "python3", "py3", ""}def _strip_code_fences(code: str) -> str:+ """Extract Python code from markdown fences, or return as-is if no fences."""code = code.strip()- match = _CODE_FENCE_PATTERN.match(code)- if match:- return match.group(1)- return code+ if "```" not in code:+ return code++ # Strip outer decorative fence pairs (e.g. ```\n```python\n...\n```\n```)+ lines = code.splitlines()+ while len(lines) >= 2 and lines[0].strip() == "```" and lines[-1].strip() == "```":+ lines.pop(0)+ lines.pop()+ code = "\n".join(lines).strip()+ if "```" not in code:+ return code++ # Find the first opening fence (skip any text before it)+ fence_start = code.find("```")+ lang_line, separator, remainder = code[fence_start + 3:].partition("\n")+ if not separator:+ return code++ # Accept python-labeled fences or bare ``` fences; reject explicit non-Python tags+ lang = (lang_line.strip().split(maxsplit=1)[0] if lang_line.strip() else "").lower()+ if lang not in _PYTHON_FENCE_LANGS:+ raise SyntaxError(f"Expected Python code but got ```{lang} fence. Write Python code, not {lang}.")++ # Find closing fence+ block_end = remainder.find("```")+ if block_end == -1:+ return remainder.strip()++ return remainder[:block_end].strip()@experimental@@ -450,7 +474,8 @@ class RLM(Module):def _process_execution_result(self,- pred: Any,+ pred: Prediction,+ code: str,result: Any,history: REPLHistory,output_field_names: list[str],@@ -461,6 +486,7 @@ class RLM(Module):Args:pred: The prediction containing reasoning and code attributes+ code: Code to record in history (already stripped when possible)result: Result from interpreter.execute() - FinalOutput, list, str, or error stringhistory: Current REPL historyoutput_field_names: List of expected output field names@@ -468,8 +494,6 @@ class RLM(Module):Returns:Prediction if FINAL was called successfully, else updated REPLHistory"""- # Strip markdown fences from code for history (format() will re-add them)- code = _strip_code_fences(pred.code)# Handle error strings from caught exceptionsif isinstance(result, str) and result.startswith("[Error]"):output = self._format_output(result)@@ -502,6 +526,18 @@ class RLM(Module):logger.info(REPLEntry.format_output(output, self.max_output_chars))return history.append(reasoning=pred.reasoning, code=code, output=output)+ def _execute_code(+ self,+ repl: CodeInterpreter,+ code: str,+ input_args: dict[str, Any],+ ) -> Any:+ """Execute code in the interpreter, returning the result or an error string."""+ try:+ return repl.execute(code, variables=dict(input_args))+ except (CodeInterpreterError, SyntaxError) as e:+ return f"[Error] {e}"+def _execute_iteration(self,repl: CodeInterpreter,@@ -526,11 +562,12 @@ class RLM(Module):try:code = _strip_code_fences(action.code)- result = repl.execute(code, variables=dict(input_args))- except (CodeInterpreterError, SyntaxError) as e:+ except SyntaxError as e:+ code = action.coderesult = f"[Error] {e}"-- return self._process_execution_result(action, result, history, output_field_names)+ return self._process_execution_result(action, code, result, history, output_field_names)+ result = self._execute_code(repl, code, input_args)+ return self._process_execution_result(action, code, result, history, output_field_names)# =========================================================================# Public Interface@@ -613,11 +650,12 @@ class RLM(Module):try:code = _strip_code_fences(pred.code)- result = repl.execute(code, variables=dict(input_args))- except (CodeInterpreterError, SyntaxError) as e:+ except SyntaxError as e:+ code = pred.coderesult = f"[Error] {e}"-- return self._process_execution_result(pred, result, history, output_field_names)+ return self._process_execution_result(pred, code, result, history, output_field_names)+ result = self._execute_code(repl, code, input_args)+ return self._process_execution_result(pred, code, result, history, output_field_names)async def aforward(self, **input_args) -> Prediction:"""Async version of forward(). Execute RLM to produce outputs.diff --git a/dspy/primitives/repl_types.py b/dspy/primitives/repl_types.pyindex d771451dc..b724a75de 100644--- a/dspy/primitives/repl_types.py+++ b/dspy/primitives/repl_types.py@@ -121,7 +121,8 @@ class REPLEntry(pydantic.BaseModel):def format(self, index: int, max_output_chars: int = 10_000) -> str:"""Format this entry for inclusion in prompts."""reasoning_line = f"Reasoning: {self.reasoning}\n" if self.reasoning else ""- return f"=== Step {index + 1} ===\n{reasoning_line}Code:\n```python\n{self.code}\n```\n{self.format_output(self.output, max_output_chars)}"+ code_block = f"```python\n{self.code}\n```"+ return f"=== Step {index + 1} ===\n{reasoning_line}Code:\n{code_block}\n{self.format_output(self.output, max_output_chars)}"class REPLHistory(pydantic.BaseModel):diff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.pyindex dab71e341..6fe70615e 100644--- a/tests/predict/test_rlm.py+++ b/tests/predict/test_rlm.py@@ -11,7 +11,7 @@ from contextlib import contextmanagerimport pytestfrom dspy.adapters.types.tool import Tool-from dspy.predict.rlm import RLM+from dspy.predict.rlm import RLM, _strip_code_fencesfrom dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutputfrom dspy.primitives.prediction import Predictionfrom dspy.primitives.python_interpreter import PythonInterpreter@@ -268,6 +268,38 @@ class TestRLMInitialization:tools["llm_query"](prompt="one more")+class TestRLMCodeFenceParsing:+ """Tests for robust fenced-code extraction."""++ @pytest.mark.parametrize(+ ("raw", "expected"),+ [+ # Standard python fence+ ("```python\nprint(1)\n```", "print(1)"),+ ("```py\nx = 1\nprint(x)\n```", "x = 1\nprint(x)"),+ # Bare fence (no language tag)+ ("```\nprint('no lang')\n```", "print('no lang')"),+ # No fences at all+ ("not fenced code", "not fenced code"),+ # Text before fence (preamble is skipped)+ ("I'll inspect first.\n```python\nprint('hello')\n```\nThen I will submit.", "print('hello')"),+ # Text after closing fence (ignored)+ ("```python\nprint(1)\n```\nsome trailing text", "print(1)"),+ # Unclosed fence (just return the body)+ ("```python\nprint('oops')", "print('oops')"),+ # Double fences (outer decorative ```)+ ("```\n```python\nprint(1)\n```\n```", "print(1)"),+ ("```\n```\nprint(2)\n```\n```", "print(2)"),+ ],+ )+ def test_strip_code_fences(self, raw, expected):+ assert _strip_code_fences(raw) == expected++ def test_strip_code_fences_rejects_non_python_lang(self):+ with pytest.raises(SyntaxError, match="json"):+ _strip_code_fences('```json\n{"a": 1}\n```')++class TestRLMFormatting:"""Tests for RLM formatting helpers."""@@ -534,6 +566,54 @@ class TestRLMToolExceptions:result = rlm.forward(query="test")assert result.answer == "recovered"+ def test_runtime_error_history_uses_stripped_code(self):+ """Runtime execution failures should preserve stripped code in history."""+ mock = MockInterpreter(responses=[+ CodeInterpreterError("NameError: name 'x' is not defined"),+ FinalOutput({"answer": "recovered"}),+ ])+ rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)+ rlm.generate_action = make_mock_predictor([+ {"reasoning": "Will fail", "code": "```python\nprint(x)\n```"},+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},+ ])++ result = rlm.forward(query="test")+ assert result.answer == "recovered"+ first_step = result.trajectory[0]+ assert first_step["code"] == "print(x)"++ def test_syntax_error_from_execute_is_recoverable(self):+ """SyntaxError from interpreter.execute should be surfaced as an iteration error."""+ mock = MockInterpreter(responses=[+ SyntaxError("invalid syntax"),+ FinalOutput({"answer": "recovered"}),+ ])+ rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)+ rlm.generate_action = make_mock_predictor([+ {"reasoning": "Bad code", "code": "```python\ndef incomplete(\n```"},+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},+ ])++ result = rlm.forward(query="test")+ assert result.answer == "recovered"+ assert result.trajectory[0]["output"].startswith("[Error] invalid syntax")++ def test_syntax_error_from_strip_code_fences_is_recoverable(self):+ """SyntaxError raised by _strip_code_fences (e.g. non-Python fence tag) should be recoverable."""+ mock = MockInterpreter(responses=[+ FinalOutput({"answer": "recovered"}),+ ])+ rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)+ rlm.generate_action = make_mock_predictor([+ {"reasoning": "Wrong language", "code": "```bash\nls -la\n```"},+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},+ ])++ result = rlm.forward(query="test")+ assert result.answer == "recovered"+ assert result.trajectory[0]["output"].startswith("[Error]")+class TestRLMDynamicSignature:"""Tests for the dynamically built RLM signatures."""