Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix(RLM): show head and tail for RLM outputs (#9282)

Isaac Millerd8fae34bdb4e
diff --git a/docs/docs/api/modules/RLM.md b/docs/docs/api/modules/RLM.md
index b957851eb..c3d550bc1 100644
--- a/docs/docs/api/modules/RLM.md
+++ b/docs/docs/api/modules/RLM.md
@@ -109,7 +109,7 @@ $20,000,000
| `signature` | `str \| Signature` | required | Defines inputs and outputs (e.g., `"context, query -> answer"`) |
| `max_iterations` | `int` | `20` | Maximum REPL interaction loops before fallback extraction |
| `max_llm_calls` | `int` | `50` | Maximum `llm_query`/`llm_query_batched` calls per execution |
-| `max_output_chars` | `int` | `100_000` | Maximum characters to include from REPL output |
+| `max_output_chars` | `int` | `10_000` | Maximum characters to include from REPL output |
| `verbose` | `bool` | `False` | Log detailed execution info |
| `tools` | `list[Union[Callable, dspy.Tool]]` | `None` | Additional tool functions callable from interpreter code |
| `sub_lm` | `dspy.LM` | `None` | LM for sub-queries. Defaults to `dspy.settings.lm`. Use a cheaper model here. |
diff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.py
index 90b1e16fd..66c011d23 100644
--- a/dspy/predict/rlm.py
+++ b/dspy/predict/rlm.py
@@ -26,7 +26,7 @@ from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreter, Code
from dspy.primitives.module import Module
from dspy.primitives.prediction import Prediction
from dspy.primitives.python_interpreter import PythonInterpreter
-from dspy.primitives.repl_types import REPLHistory, REPLVariable
+from dspy.primitives.repl_types import REPLEntry, REPLHistory, REPLVariable
from dspy.signatures.signature import ensure_signature
from dspy.utils.annotation import experimental
@@ -103,7 +103,7 @@ class RLM(Module):
signature: type[Signature] | str,
max_iterations: int = 20,
max_llm_calls: int = 50,
- max_output_chars: int = 100_000,
+ max_output_chars: int = 10_000,
verbose: bool = False,
tools: list[Callable] | None = None,
sub_lm: dspy.LM | None = None,
@@ -336,11 +336,8 @@ class RLM(Module):
return variables
def _format_output(self, output: str) -> str:
- """Format and truncate REPL output."""
if not output:
return "(no output - did you forget to print?)"
- if len(output) > self.max_output_chars:
- return output[:self.max_output_chars] + "\n... (truncated)"
return output
def _validate_inputs(self, input_args: dict[str, Any]) -> None:
@@ -501,6 +498,8 @@ class RLM(Module):
output = str(result) if result else ""
output = self._format_output(output)
+ if self.verbose:
+ logger.info(REPLEntry.format_output(output, self.max_output_chars))
return history.append(reasoning=pred.reasoning, code=code, output=output)
def _execute_iteration(
@@ -556,7 +555,7 @@ class RLM(Module):
variables = self._build_variables(**input_args)
with self._interpreter_context(execution_tools) as repl:
- history: REPLHistory = REPLHistory()
+ history: REPLHistory = REPLHistory(max_output_chars=self.max_output_chars)
for iteration in range(self.max_iterations):
result: Prediction | REPLHistory = self._execute_iteration(
@@ -639,7 +638,7 @@ class RLM(Module):
variables = self._build_variables(**input_args)
with self._interpreter_context(execution_tools) as repl:
- history = REPLHistory()
+ history = REPLHistory(max_output_chars=self.max_output_chars)
for iteration in range(self.max_iterations):
result = await self._aexecute_iteration(
diff --git a/dspy/primitives/repl_types.py b/dspy/primitives/repl_types.py
index d13af7940..aef5bd52e 100644
--- a/dspy/primitives/repl_types.py
+++ b/dspy/primitives/repl_types.py
@@ -57,7 +57,11 @@ class REPLVariable(pydantic.BaseModel):
else:
value_str = str(jsonable)
is_truncated = len(value_str) > preview_chars
- preview = value_str[:preview_chars] + ("..." if is_truncated else "")
+ if is_truncated:
+ half = preview_chars // 2
+ preview = value_str[:half] + "..." + value_str[-half:]
+ else:
+ preview = value_str
# Extract desc and constraints from field_info if provided
desc = ""
@@ -104,13 +108,20 @@ class REPLEntry(pydantic.BaseModel):
model_config = pydantic.ConfigDict(frozen=True)
- def format(self, index: int, max_output_chars: int = 5000) -> str:
+ @staticmethod
+ def format_output(output: str, max_output_chars: int = 10_000) -> str:
+ """Format output with head+tail truncation, preserving true length in header."""
+ raw_len = len(output)
+ if raw_len > max_output_chars:
+ half = max_output_chars // 2
+ omitted = raw_len - max_output_chars
+ output = output[:half] + f"\n\n... ({omitted:,} characters omitted) ...\n\n" + output[-half:]
+ return f"Output ({raw_len:,} chars):\n{output}"
+
+ def format(self, index: int, max_output_chars: int = 10_000) -> str:
"""Format this entry for inclusion in prompts."""
- output = self.output
- if len(output) > max_output_chars:
- output = output[:max_output_chars] + f"\n... (truncated to {max_output_chars}/{len(self.output):,} chars)"
reasoning_line = f"Reasoning: {self.reasoning}\n" if self.reasoning else ""
- return f"=== Step {index + 1} ===\n{reasoning_line}Code:\n```python\n{self.code}\n```\nOutput ({len(self.output):,} chars):\n{output}"
+ return f"=== Step {index + 1} ===\n{reasoning_line}Code:\n```python\n{self.code}\n```\n{self.format_output(self.output, max_output_chars)}"
class REPLHistory(pydantic.BaseModel):
@@ -120,13 +131,14 @@ class REPLHistory(pydantic.BaseModel):
"""
entries: list[REPLEntry] = Field(default_factory=list)
+ max_output_chars: int = 10_000
model_config = pydantic.ConfigDict(frozen=True)
- def format(self, max_output_chars: int = 5000) -> str:
+ def format(self) -> str:
if not self.entries:
return "You have not interacted with the REPL environment yet."
- return "\n".join(entry.format(index=i, max_output_chars=max_output_chars) for i, entry in enumerate(self.entries))
+ return "\n".join(entry.format(index=i, max_output_chars=self.max_output_chars) for i, entry in enumerate(self.entries))
@pydantic.model_serializer()
def serialize_model(self) -> str:
@@ -135,7 +147,7 @@ class REPLHistory(pydantic.BaseModel):
def append(self, *, reasoning: str = "", code: str, output: str) -> REPLHistory:
"""Return a new REPLHistory with the entry appended."""
new_entry = REPLEntry(reasoning=reasoning, code=code, output=output)
- return REPLHistory(entries=list(self.entries) + [new_entry])
+ return REPLHistory(entries=list(self.entries) + [new_entry], max_output_chars=self.max_output_chars)
def __len__(self) -> int:
return len(self.entries)
diff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.py
index 3e54d3249..dab71e341 100644
--- a/tests/predict/test_rlm.py
+++ b/tests/predict/test_rlm.py
@@ -306,11 +306,12 @@ class TestRLMFormatting:
formatted = rlm._format_output("")
assert "no output" in formatted.lower()
- def test_format_output_truncation(self):
- """Test that long output is truncated."""
+ def test_format_output_passthrough(self):
+ """Test that _format_output passes through non-empty output without truncation."""
rlm = RLM("context -> answer", max_output_chars=100)
- formatted = rlm._format_output("x" * 200)
- assert "truncated" in formatted.lower()
+ long_output = "a" * 200
+ formatted = rlm._format_output(long_output)
+ assert formatted == long_output
def test_format_variable_info_string(self):
"""Test variable info formatting for string value using REPLVariable."""
@@ -319,7 +320,8 @@ class TestRLMFormatting:
assert "Variable: `context`" in formatted
assert "Type: str" in formatted
assert "11" in formatted # length
- assert "Hello" in formatted
+ assert "He" in formatted # head
+ assert "ld" in formatted # tail
assert "..." in formatted # truncation indicator
def test_format_variable_info_dict(self):
@@ -379,10 +381,33 @@ class TestREPLTypes:
assert "1" in formatted
def test_repl_entry_format_truncation(self):
- """Test REPLEntry output truncation."""
- entry = REPLEntry(code="print('x' * 1000)", output="x" * 1000)
- formatted = entry.format(index=0, max_output_chars=50)
- assert "truncated" in formatted
+ """Test REPLEntry.format() truncates with head+tail and shows true length."""
+ output = "a" * 100 + "b" * 100
+ entry = REPLEntry(code="print('a' + 'b')", output=output)
+ formatted = entry.format(index=0, max_output_chars=100)
+ # Head and tail preserved
+ assert "a" * 50 in formatted
+ assert "b" * 50 in formatted
+ assert "100 characters omitted" in formatted
+ # True original length shown in header
+ assert "200 chars" in formatted
+
+ def test_repl_entry_format_no_truncation(self):
+ """Test REPLEntry.format() passes short output through without truncation."""
+ output = "a" * 50
+ entry = REPLEntry(code="print('a')", output=output)
+ formatted = entry.format(index=0, max_output_chars=100)
+ assert output in formatted
+ assert "omitted" not in formatted
+
+ def test_repl_history_threads_max_output_chars(self):
+ """Test REPLHistory carries max_output_chars through append()."""
+ h = REPLHistory(max_output_chars=50)
+ h2 = h.append(code="print(1)", output="a" * 100)
+ assert h2.max_output_chars == 50
+ # Formatting should truncate at 50 chars
+ formatted = h2.format()
+ assert "50 characters omitted" in formatted
def test_repl_variable_from_value(self):
"""Test REPLVariable.from_value() factory."""
@@ -393,10 +418,11 @@ class TestREPLTypes:
assert "hello world" in var.preview
def test_repl_variable_truncation(self):
- """Test REPLVariable preview truncation."""
- var = REPLVariable.from_value("big", "x" * 1000, preview_chars=50)
- assert len(var.preview) == 53 # 50 + "..."
- assert var.preview.endswith("...")
+ """Test REPLVariable preview shows head and tail."""
+ var = REPLVariable.from_value("big", "a" * 500 + "b" * 500, preview_chars=50)
+ assert var.preview.startswith("a" * 25)
+ assert var.preview.endswith("b" * 25)
+ assert "..." in var.preview
def test_repl_variable_with_field_info(self):
"""Test REPLVariable includes desc and constraints from field_info."""