Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

refactor(rlm): enhance code fence parsing and REPL entry formatting (#9309)

Isaac Miller3109c61fa4e7
diff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.py
index 66c011d23..48f55f79e 100644
--- a/dspy/predict/rlm.py
+++ b/dspy/predict/rlm.py
@@ -11,7 +11,6 @@ Reference: "Recursive Language Models" (Zhang, Kraska, Khattab, 2025)
from __future__ import annotations
import logging
-import re
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import contextmanager
@@ -62,16 +61,41 @@ IMPORTANT: This is ITERATIVE. Each code block you write will execute, you'll see
You have max {max_llm_calls} sub-LLM calls. When done, call SUBMIT() with your output."""
-# Pattern to match markdown code fences: ```python\n...\n``` or ```\n...\n```
-_CODE_FENCE_PATTERN = re.compile(r"^```(?:python|py)?\s*\n(.*)\n```\s*$", re.DOTALL)
+_PYTHON_FENCE_LANGS = {"python", "py", "python3", "py3", ""}
def _strip_code_fences(code: str) -> str:
+ """Extract Python code from markdown fences, or return as-is if no fences."""
code = code.strip()
- match = _CODE_FENCE_PATTERN.match(code)
- if match:
- return match.group(1)
- return code
+ if "```" not in code:
+ return code
+
+ # Strip outer decorative fence pairs (e.g. ```\n```python\n...\n```\n```)
+ lines = code.splitlines()
+ while len(lines) >= 2 and lines[0].strip() == "```" and lines[-1].strip() == "```":
+ lines.pop(0)
+ lines.pop()
+ code = "\n".join(lines).strip()
+ if "```" not in code:
+ return code
+
+ # Find the first opening fence (skip any text before it)
+ fence_start = code.find("```")
+ lang_line, separator, remainder = code[fence_start + 3:].partition("\n")
+ if not separator:
+ return code
+
+ # Accept python-labeled fences or bare ``` fences; reject explicit non-Python tags
+ lang = (lang_line.strip().split(maxsplit=1)[0] if lang_line.strip() else "").lower()
+ if lang not in _PYTHON_FENCE_LANGS:
+ raise SyntaxError(f"Expected Python code but got ```{lang} fence. Write Python code, not {lang}.")
+
+ # Find closing fence
+ block_end = remainder.find("```")
+ if block_end == -1:
+ return remainder.strip()
+
+ return remainder[:block_end].strip()
@experimental
@@ -450,7 +474,8 @@ class RLM(Module):
def _process_execution_result(
self,
- pred: Any,
+ pred: Prediction,
+ code: str,
result: Any,
history: REPLHistory,
output_field_names: list[str],
@@ -461,6 +486,7 @@ class RLM(Module):
Args:
pred: The prediction containing reasoning and code attributes
+ code: Code to record in history (already stripped when possible)
result: Result from interpreter.execute() - FinalOutput, list, str, or error string
history: Current REPL history
output_field_names: List of expected output field names
@@ -468,8 +494,6 @@ class RLM(Module):
Returns:
Prediction if FINAL was called successfully, else updated REPLHistory
"""
- # Strip markdown fences from code for history (format() will re-add them)
- code = _strip_code_fences(pred.code)
# Handle error strings from caught exceptions
if isinstance(result, str) and result.startswith("[Error]"):
output = self._format_output(result)
@@ -502,6 +526,18 @@ class RLM(Module):
logger.info(REPLEntry.format_output(output, self.max_output_chars))
return history.append(reasoning=pred.reasoning, code=code, output=output)
+ def _execute_code(
+ self,
+ repl: CodeInterpreter,
+ code: str,
+ input_args: dict[str, Any],
+ ) -> Any:
+ """Execute code in the interpreter, returning the result or an error string."""
+ try:
+ return repl.execute(code, variables=dict(input_args))
+ except (CodeInterpreterError, SyntaxError) as e:
+ return f"[Error] {e}"
+
def _execute_iteration(
self,
repl: CodeInterpreter,
@@ -526,11 +562,12 @@ class RLM(Module):
try:
code = _strip_code_fences(action.code)
- result = repl.execute(code, variables=dict(input_args))
- except (CodeInterpreterError, SyntaxError) as e:
+ except SyntaxError as e:
+ code = action.code
result = f"[Error] {e}"
-
- return self._process_execution_result(action, result, history, output_field_names)
+ return self._process_execution_result(action, code, result, history, output_field_names)
+ result = self._execute_code(repl, code, input_args)
+ return self._process_execution_result(action, code, result, history, output_field_names)
# =========================================================================
# Public Interface
@@ -613,11 +650,12 @@ class RLM(Module):
try:
code = _strip_code_fences(pred.code)
- result = repl.execute(code, variables=dict(input_args))
- except (CodeInterpreterError, SyntaxError) as e:
+ except SyntaxError as e:
+ code = pred.code
result = f"[Error] {e}"
-
- return self._process_execution_result(pred, result, history, output_field_names)
+ return self._process_execution_result(pred, code, result, history, output_field_names)
+ result = self._execute_code(repl, code, input_args)
+ return self._process_execution_result(pred, code, result, history, output_field_names)
async def aforward(self, **input_args) -> Prediction:
"""Async version of forward(). Execute RLM to produce outputs.
diff --git a/dspy/primitives/repl_types.py b/dspy/primitives/repl_types.py
index d771451dc..b724a75de 100644
--- a/dspy/primitives/repl_types.py
+++ b/dspy/primitives/repl_types.py
@@ -121,7 +121,8 @@ class REPLEntry(pydantic.BaseModel):
def format(self, index: int, max_output_chars: int = 10_000) -> str:
"""Format this entry for inclusion in prompts."""
reasoning_line = f"Reasoning: {self.reasoning}\n" if self.reasoning else ""
- return f"=== Step {index + 1} ===\n{reasoning_line}Code:\n```python\n{self.code}\n```\n{self.format_output(self.output, max_output_chars)}"
+ code_block = f"```python\n{self.code}\n```"
+ return f"=== Step {index + 1} ===\n{reasoning_line}Code:\n{code_block}\n{self.format_output(self.output, max_output_chars)}"
class REPLHistory(pydantic.BaseModel):
diff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.py
index dab71e341..6fe70615e 100644
--- a/tests/predict/test_rlm.py
+++ b/tests/predict/test_rlm.py
@@ -11,7 +11,7 @@ from contextlib import contextmanager
import pytest
from dspy.adapters.types.tool import Tool
-from dspy.predict.rlm import RLM
+from dspy.predict.rlm import RLM, _strip_code_fences
from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutput
from dspy.primitives.prediction import Prediction
from dspy.primitives.python_interpreter import PythonInterpreter
@@ -268,6 +268,38 @@ class TestRLMInitialization:
tools["llm_query"](prompt="one more")
+class TestRLMCodeFenceParsing:
+ """Tests for robust fenced-code extraction."""
+
+ @pytest.mark.parametrize(
+ ("raw", "expected"),
+ [
+ # Standard python fence
+ ("```python\nprint(1)\n```", "print(1)"),
+ ("```py\nx = 1\nprint(x)\n```", "x = 1\nprint(x)"),
+ # Bare fence (no language tag)
+ ("```\nprint('no lang')\n```", "print('no lang')"),
+ # No fences at all
+ ("not fenced code", "not fenced code"),
+ # Text before fence (preamble is skipped)
+ ("I'll inspect first.\n```python\nprint('hello')\n```\nThen I will submit.", "print('hello')"),
+ # Text after closing fence (ignored)
+ ("```python\nprint(1)\n```\nsome trailing text", "print(1)"),
+ # Unclosed fence (just return the body)
+ ("```python\nprint('oops')", "print('oops')"),
+ # Double fences (outer decorative ```)
+ ("```\n```python\nprint(1)\n```\n```", "print(1)"),
+ ("```\n```\nprint(2)\n```\n```", "print(2)"),
+ ],
+ )
+ def test_strip_code_fences(self, raw, expected):
+ assert _strip_code_fences(raw) == expected
+
+ def test_strip_code_fences_rejects_non_python_lang(self):
+ with pytest.raises(SyntaxError, match="json"):
+ _strip_code_fences('```json\n{"a": 1}\n```')
+
+
class TestRLMFormatting:
"""Tests for RLM formatting helpers."""
@@ -534,6 +566,54 @@ class TestRLMToolExceptions:
result = rlm.forward(query="test")
assert result.answer == "recovered"
+ def test_runtime_error_history_uses_stripped_code(self):
+ """Runtime execution failures should preserve stripped code in history."""
+ mock = MockInterpreter(responses=[
+ CodeInterpreterError("NameError: name 'x' is not defined"),
+ FinalOutput({"answer": "recovered"}),
+ ])
+ rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)
+ rlm.generate_action = make_mock_predictor([
+ {"reasoning": "Will fail", "code": "```python\nprint(x)\n```"},
+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},
+ ])
+
+ result = rlm.forward(query="test")
+ assert result.answer == "recovered"
+ first_step = result.trajectory[0]
+ assert first_step["code"] == "print(x)"
+
+ def test_syntax_error_from_execute_is_recoverable(self):
+ """SyntaxError from interpreter.execute should be surfaced as an iteration error."""
+ mock = MockInterpreter(responses=[
+ SyntaxError("invalid syntax"),
+ FinalOutput({"answer": "recovered"}),
+ ])
+ rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)
+ rlm.generate_action = make_mock_predictor([
+ {"reasoning": "Bad code", "code": "```python\ndef incomplete(\n```"},
+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},
+ ])
+
+ result = rlm.forward(query="test")
+ assert result.answer == "recovered"
+ assert result.trajectory[0]["output"].startswith("[Error] invalid syntax")
+
+ def test_syntax_error_from_strip_code_fences_is_recoverable(self):
+ """SyntaxError raised by _strip_code_fences (e.g. non-Python fence tag) should be recoverable."""
+ mock = MockInterpreter(responses=[
+ FinalOutput({"answer": "recovered"}),
+ ])
+ rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)
+ rlm.generate_action = make_mock_predictor([
+ {"reasoning": "Wrong language", "code": "```bash\nls -la\n```"},
+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},
+ ])
+
+ result = rlm.forward(query="test")
+ assert result.answer == "recovered"
+ assert result.trajectory[0]["output"].startswith("[Error]")
+
class TestRLMDynamicSignature:
"""Tests for the dynamically built RLM signatures."""