DSPy: The framework for programming—not prompting—language models
Python
Change FinalAnswerResult to FinalOutput and remove RLM __call__ (#9212)
Isaac Millerb65bf378e5e0
diff --git a/dspy/predict/program_of_thought.py b/dspy/predict/program_of_thought.pyindex 44f9d34e1..374806016 100644--- a/dspy/predict/program_of_thought.py+++ b/dspy/predict/program_of_thought.py@@ -3,7 +3,7 @@ import loggingimport reimport dspy-from dspy.primitives.code_interpreter import FinalAnswerResult+from dspy.primitives.code_interpreter import FinalOutputfrom dspy.primitives.module import Modulefrom dspy.primitives.python_interpreter import PythonInterpreterfrom dspy.signatures.signature import Signature, ensure_signature@@ -53,7 +53,7 @@ class ProgramOfThought(Module):self._generate_instruction("regenerate"),),)- self.generate_answer = dspy.ChainOfThought(+ self.generate_output = dspy.ChainOfThought(dspy.Signature(self._generate_signature("answer").fields,self._generate_instruction("answer"),@@ -167,8 +167,8 @@ class ProgramOfThought(Module):try:result = self.interpreter.execute(code)- if isinstance(result, FinalAnswerResult):- result = result.answer+ if isinstance(result, FinalOutput):+ result = result.output# Since it's more complex structure now, just blindly use json to represents all.output = json.dumps(result)return output, None@@ -196,6 +196,6 @@ class ProgramOfThought(Module):output, error = self._execute_code(code)hop += 1input_kwargs.update({"final_generated_code": code, "code_output": output})- answer_gen_result = self.generate_answer(**input_kwargs)+ output_gen_result = self.generate_output(**input_kwargs)self.interpreter.shutdown()- return answer_gen_result+ return output_gen_resultdiff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.pyindex 7e27d1ef7..48fef4da4 100644--- a/dspy/predict/rlm.py+++ b/dspy/predict/rlm.py@@ -22,7 +22,7 @@ import pydanticimport dspyfrom dspy.adapters.utils import parse_value, translate_field_type-from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreter, CodeInterpreterError, FinalAnswerResult+from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreter, CodeInterpreterError, FinalOutputfrom dspy.primitives.module import Modulefrom dspy.primitives.prediction import Predictionfrom dspy.primitives.python_interpreter import PythonInterpreter@@ -92,9 +92,9 @@ class RLM(Module):Example:```python# Basic usage- rlm = dspy.RLM("context, query -> answer", max_iterations=10)+ rlm = dspy.RLM("context, query -> output", max_iterations=10)result = rlm(context="...very long text...", query="What is the magic number?")- print(result.answer)+ print(result.output)```"""@@ -386,7 +386,7 @@ class RLM(Module):history: REPLHistory,output_field_names: list[str],) -> Prediction:- """Use extract module to get final answer when max iterations reached."""+ """Use extract module to get final output when max iterations reached."""logger.warning("RLM reached max iterations, using extract to get final output")variables_info = [variable.format() for variable in variables]@@ -397,24 +397,24 @@ class RLM(Module):return Prediction(trajectory=[e.model_dump() for e in history],- final_reasoning="Extract forced final answer",+ final_reasoning="Extract forced final output",**{name: getattr(extract_pred, name) for name in output_field_names},)- def _process_final_answer(+ def _process_final_output(self,- result: FinalAnswerResult,+ result: FinalOutput,output_field_names: list[str],) -> tuple[dict[str, Any] | None, str | None]:- """Validate and parse FinalAnswerResult. Returns (parsed_outputs, None) or (None, error)."""- raw_answer = result.answer+ """Validate and parse FinalOutput. Returns (parsed_outputs, None) or (None, error)."""+ raw_output = result.output- # Validate raw_answer is a dict- if not isinstance(raw_answer, dict):- return None, f"[Error] FINAL returned {type(raw_answer).__name__}, expected dict with fields: {output_field_names}"+ # Validate raw_output is a dict+ if not isinstance(raw_output, dict):+ return None, f"[Error] FINAL returned {type(raw_output).__name__}, expected dict with fields: {output_field_names}"# Validate all required output fields are present- missing = set(output_field_names) - set(raw_answer.keys())+ missing = set(output_field_names) - set(raw_output.keys())if missing:return None, f"[Error] Missing output fields: {sorted(missing)}. Use SUBMIT({', '.join(output_field_names)})"@@ -425,11 +425,11 @@ class RLM(Module):field = self.signature.output_fields[name]annotation = getattr(field, "annotation", str)try:- parsed_outputs[name] = parse_value(raw_answer[name], annotation)+ parsed_outputs[name] = parse_value(raw_output[name], annotation)except (ValueError, pydantic.ValidationError) as e:type_errors.append(f"{name}: expected {annotation.__name__ if hasattr(annotation, '__name__') else annotation}, "- f"got {type(raw_answer[name]).__name__}: {e}"+ f"got {type(raw_output[name]).__name__}: {e}")if type_errors:@@ -450,7 +450,7 @@ class RLM(Module):Args:pred: The prediction containing reasoning and code attributes- result: Result from interpreter.execute() - FinalAnswerResult, list, str, or error string+ result: Result from interpreter.execute() - FinalOutput, list, str, or error stringhistory: Current REPL historyoutput_field_names: List of expected output field names@@ -462,9 +462,9 @@ class RLM(Module):output = self._format_output(result)return history.append(reasoning=pred.reasoning, code=pred.code, output=output)- # Handle FINAL answer- if isinstance(result, FinalAnswerResult):- parsed_outputs, error = self._process_final_answer(result, output_field_names)+ # Handle FINAL output+ if isinstance(result, FinalOutput):+ parsed_outputs, error = self._process_final_output(result, output_field_names)if error:return history.append(reasoning=pred.reasoning, code=pred.code, output=error)@@ -570,7 +570,7 @@ class RLM(Module):return Prediction(trajectory=[e.model_dump() for e in history],- final_reasoning="Extract forced final answer",+ final_reasoning="Extract forced final output",**{name: getattr(extract_pred, name) for name in output_field_names},)@@ -635,7 +635,3 @@ class RLM(Module):# Max iterations reached - use extract fallbackreturn await self._aextract_fallback(variables, history, output_field_names)-- def __call__(self, **input_args) -> Prediction:- """Execute RLM. Alias for forward()."""- return self.forward(**input_args)diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.pyindex b45a0da2e..367394aa4 100644--- a/dspy/primitives/__init__.py+++ b/dspy/primitives/__init__.py@@ -1,5 +1,5 @@from dspy.primitives.base_module import BaseModule-from dspy.primitives.code_interpreter import CodeInterpreter, CodeInterpreterError, FinalAnswerResult+from dspy.primitives.code_interpreter import CodeInterpreter, CodeInterpreterError, FinalOutputfrom dspy.primitives.example import Examplefrom dspy.primitives.module import Modulefrom dspy.primitives.prediction import Completions, Prediction@@ -10,7 +10,7 @@ __all__ = ["CodeInterpreter","Completions","Example",- "FinalAnswerResult",+ "FinalOutput","CodeInterpreterError","Module","Prediction",diff --git a/dspy/primitives/code_interpreter.py b/dspy/primitives/code_interpreter.pyindex 3838074fe..030503818 100644--- a/dspy/primitives/code_interpreter.py+++ b/dspy/primitives/code_interpreter.py@@ -21,23 +21,23 @@ class CodeInterpreterError(RuntimeError):"""-class FinalAnswerResult:+class FinalOutput:"""Returned by interpreter.execute() when SUBMIT() is called.This signals that the code execution loop should terminate and return- the contained answer to the caller.+ the contained output to the caller."""- def __init__(self, answer: Any):- self.answer = answer+ def __init__(self, output: Any):+ self.output = outputdef __repr__(self) -> str:- return f"FinalAnswerResult({self.answer!r})"+ return f"FinalOutput({self.output!r})"def __eq__(self, other: object) -> bool:- if not isinstance(other, FinalAnswerResult):+ if not isinstance(other, FinalOutput):return NotImplemented- return self.answer == other.answer+ return self.output == other.output@runtime_checkable@@ -107,7 +107,7 @@ class CodeInterpreter(Protocol):Returns:One of:- - FinalAnswerResult: If SUBMIT() was called in code+ - FinalOutput: If SUBMIT() was called in code- str: Captured stdout from print() statements- list: Multiple output lines- None: If no output was produceddiff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex b492058db..3136c4a08 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -16,9 +16,9 @@ import subprocessfrom os import PathLikefrom typing import Any, Callable-from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreterError, FinalAnswerResult+from dspy.primitives.code_interpreter import SIMPLE_TYPES, CodeInterpreterError, FinalOutput-__all__ = ["PythonInterpreter", "FinalAnswerResult", "CodeInterpreterError"]+__all__ = ["PythonInterpreter", "FinalOutput", "CodeInterpreterError"]logger = logging.getLogger(__name__)@@ -361,10 +361,10 @@ class PythonInterpreter:error_type = result.get("errorType", "Sandbox Error")error_args = result.get("errorArgs", [])- if error_type == "FinalAnswer":- answer = error_args[0] if error_args else None+ if error_type == "FinalOutput":+ output = error_args[0] if error_args else Noneself._sync_files()- return FinalAnswerResult(answer)+ return FinalOutput(output)elif error_type == "SyntaxError":raise SyntaxError(f"Invalid Python syntax. message: {error_msg}")else:diff --git a/dspy/primitives/runner.js b/dspy/primitives/runner.jsindex 28d9f5a25..b2dca376d 100644--- a/dspy/primitives/runner.js+++ b/dspy/primitives/runner.js@@ -19,15 +19,15 @@ sys.stdout, sys.stderr = buf_stdout, buf_stderrdef last_exception_args():return json.dumps(sys.last_exc.args) if sys.last_exc else None-class FinalAnswer(BaseException):+class FinalOutput(BaseException):# Control-flow exception to signal completion (like StopIteration)pass# Default SUBMIT for single-output signatures (e.g., Program of Thought).# Only define if not already registered with typed signatures.if 'SUBMIT' not in dir():- def SUBMIT(answer):- raise FinalAnswer({"answer": answer})+ def SUBMIT(output):+ raise FinalOutput({"output": output})`;// Generate a tool wrapper function with typed signature.@@ -78,8 +78,8 @@ const makeSubmitWrapper = (outputs) => {if (!outputs || outputs.length === 0) {// Fallback to single-arg SUBMIT if no outputs definedreturn `-def SUBMIT(answer):- raise FinalAnswer({"answer": answer})+def SUBMIT(output):+ raise FinalOutput({"output": output})`;}@@ -92,7 +92,7 @@ def SUBMIT(answer):return `def SUBMIT(${sigParts.join(', ')}):- raise FinalAnswer({${dictParts.join(', ')}})+ raise FinalOutput({${dictParts.join(', ')}})`;};diff --git a/tests/mock_interpreter.py b/tests/mock_interpreter.pyindex e36ceecf1..255f0282f 100644--- a/tests/mock_interpreter.py+++ b/tests/mock_interpreter.py@@ -10,7 +10,7 @@ or uses a custom function to generate responses. Useful for:from typing import Any, Callable-from dspy.primitives.code_interpreter import CodeInterpreterError, FinalAnswerResult+from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutput__all__ = ["MockInterpreter"]@@ -25,15 +25,15 @@ class MockInterpreter:# Script specific responsesmock = MockInterpreter(responses=["data explored",- FinalAnswerResult("42"),+ FinalOutput("42"),])result1 = mock.execute("print(len(context))") # Returns "data explored"- result2 = mock.execute("SUBMIT('42')") # Returns FinalAnswerResult("42")+ result2 = mock.execute("SUBMIT('42')") # Returns FinalOutput("42")# Use custom execution functiondef custom_exec(code, variables):if "SUBMIT" in code:- return FinalAnswerResult("done")+ return FinalOutput("done")return f"executed: {code[:20]}..."mock = MockInterpreter(execute_fn=custom_exec)@@ -42,7 +42,7 @@ class MockInterpreter:def __init__(self,- responses: list[str | FinalAnswerResult | Exception] | None = None,+ responses: list[str | FinalOutput | Exception] | None = None,execute_fn: Callable[[str, dict[str, Any]], Any] | None = None,tools: dict[str, Callable[..., str]] | None = None,):diff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.pyindex fbbab742e..54327c1af 100644--- a/tests/predict/test_rlm.py+++ b/tests/predict/test_rlm.py@@ -11,7 +11,7 @@ from contextlib import contextmanagerimport pytestfrom dspy.predict.rlm import RLM-from dspy.primitives.code_interpreter import CodeInterpreterError, FinalAnswerResult+from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutputfrom dspy.primitives.prediction import Predictionfrom dspy.primitives.python_interpreter import PythonInterpreterfrom dspy.primitives.repl_types import REPLEntry, REPLHistory, REPLVariable@@ -89,13 +89,13 @@ class TestMockInterpreter:assert mock.execute("code2") == "second"assert mock.execute("code3") == "third"- def test_returns_final_answer_result(self):- """Test that MockInterpreter can return FinalAnswerResult."""- mock = MockInterpreter(responses=["exploring", FinalAnswerResult("42")])+ def test_returns_final_output_result(self):+ """Test that MockInterpreter can return FinalOutput."""+ mock = MockInterpreter(responses=["exploring", FinalOutput("42")])assert mock.execute("print(len(data))") == "exploring"result = mock.execute("SUBMIT('42')")- assert isinstance(result, FinalAnswerResult)- assert result.answer == "42"+ assert isinstance(result, FinalOutput)+ assert result.output == "42"def test_raises_exception_from_responses(self):"""Test that MockInterpreter raises exceptions from responses."""@@ -440,7 +440,7 @@ class TestRLMCallMethod:def test_call_is_alias_for_forward(self):"""Test that __call__ is an alias for forward()."""- mock = MockInterpreter(responses=[FinalAnswerResult({"answer": "42"})])+ mock = MockInterpreter(responses=[FinalOutput({"answer": "42"})])rlm = RLM("query -> answer", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([{"reasoning": "Return answer", "code": 'SUBMIT("42")'},@@ -473,7 +473,7 @@ class TestRLMMaxIterationsFallback:result = rlm.forward(query="test")assert result.answer == "extracted_answer"- assert result.final_reasoning == "Extract forced final answer"+ assert result.final_reasoning == "Extract forced final output"class TestRLMToolExceptions:@@ -486,7 +486,7 @@ class TestRLMToolExceptions:mock = MockInterpreter(responses=[CodeInterpreterError("RuntimeError: Tool failed!"),- FinalAnswerResult({"answer": "recovered"}),+ FinalOutput({"answer": "recovered"}),])rlm = RLM("query -> answer", max_iterations=5, interpreter=mock, tools={"failing_tool": failing_tool})rlm.generate_action = make_mock_predictor([@@ -712,7 +712,7 @@ class TestRLMAsyncMock:@pytest.mark.asyncioasync def test_aforward_basic(self):"""Test aforward() returns Prediction with expected output (MockInterpreter)."""- mock = MockInterpreter(responses=[FinalAnswerResult({"answer": "42"})])+ mock = MockInterpreter(responses=[FinalOutput({"answer": "42"})])rlm = RLM("query -> answer", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([{"reasoning": "Return answer", "code": 'SUBMIT("42")'},@@ -724,7 +724,7 @@ class TestRLMAsyncMock:@pytest.mark.asyncioasync def test_aforward_int_output_mock(self):"""Test aforward() returns int when signature expects int (MockInterpreter)."""- mock = MockInterpreter(responses=[FinalAnswerResult({"count": 42})])+ mock = MockInterpreter(responses=[FinalOutput({"count": 42})])rlm = RLM("query -> count: int", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([{"reasoning": "Return count", "code": "SUBMIT(42)"},@@ -739,7 +739,7 @@ class TestRLMAsyncMock:"""Test aforward() handles multiple iterations before SUBMIT (MockInterpreter)."""mock = MockInterpreter(responses=["explored data",- FinalAnswerResult({"answer": "done"}),+ FinalOutput({"answer": "done"}),])rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)rlm.generate_action = make_mock_predictor([@@ -763,7 +763,7 @@ class TestRLMTypeCoercionMock:])def test_type_coercion(self, output_field, output_type, final_value, code, expected):"""Test RLM type coercion for various types (MockInterpreter)."""- mock = MockInterpreter(responses=[FinalAnswerResult({output_field: final_value})])+ mock = MockInterpreter(responses=[FinalOutput({output_field: final_value})])rlm = RLM(f"query -> {output_field}: {output_type}", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([{"reasoning": "Return value", "code": code},@@ -775,8 +775,8 @@ class TestRLMTypeCoercionMock:def test_type_error_retries(self):"""Test RLM retries when type validation fails (MockInterpreter)."""mock = MockInterpreter(responses=[- FinalAnswerResult({"answer": "maybe"}), # Invalid for Literal- FinalAnswerResult({"answer": "yes"}), # Valid+ FinalOutput({"answer": "maybe"}), # Invalid for Literal+ FinalOutput({"answer": "yes"}), # Valid])rlm = RLM("query -> answer: Literal['yes', 'no']", max_iterations=5, interpreter=mock)rlm.generate_action = make_mock_predictor([diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.pyindex fef4965e1..40b4d1b6d 100644--- a/tests/primitives/test_python_interpreter.py+++ b/tests/primitives/test_python_interpreter.py@@ -4,7 +4,7 @@ import shutilimport pytest-from dspy.primitives.code_interpreter import CodeInterpreterError, FinalAnswerResult+from dspy.primitives.code_interpreter import CodeInterpreterError, FinalOutputfrom dspy.primitives.python_interpreter import PythonInterpreter# This test suite requires deno to be installed. Please install deno following https://docs.deno.com/runtime/getting_started/installation/@@ -68,16 +68,16 @@ def test_exception_args():def test_submit_with_list():- """Test SUBMIT() with a list argument returns FinalAnswerResult with dict format."""+ """Test SUBMIT() with a list argument returns FinalOutput with dict format."""with PythonInterpreter() as interpreter:token = random.randint(1, 10**9)code = f"SUBMIT(['The result is', {token}])"result = interpreter(code)- assert isinstance(result, FinalAnswerResult)- # SUBMIT now always returns a dict with "answer" key for single-output default- assert result.answer == {"answer": ["The result is", token]}+ assert isinstance(result, FinalOutput)+ # SUBMIT now always returns a dict with "output" key for single-output default+ assert result.output == {"output": ["The result is", token]}def test_enable_env_vars_flag():os.environ["FOO_TEST_ENV"] = "test_value"@@ -326,8 +326,8 @@ def test_submit_with_typed_signature():with PythonInterpreter(output_fields=output_fields) as sandbox:result = sandbox.execute('SUBMIT(answer="the answer", confidence=0.95)')- assert isinstance(result, FinalAnswerResult)- assert result.answer == {"answer": "the answer", "confidence": 0.95}+ assert isinstance(result, FinalOutput)+ assert result.output == {"answer": "the answer", "confidence": 0.95}def test_submit_positional_args():@@ -341,8 +341,8 @@ def test_submit_positional_args():with PythonInterpreter(output_fields=output_fields) as sandbox:result = sandbox.execute('SUBMIT("the answer", 0.95)')- assert isinstance(result, FinalAnswerResult)- assert result.answer == {"answer": "the answer", "confidence": 0.95}+ assert isinstance(result, FinalOutput)+ assert result.output == {"answer": "the answer", "confidence": 0.95}def test_submit_multi_output():@@ -362,8 +362,8 @@ SUBMIT(a, s)"""result = sandbox.execute(code)- assert isinstance(result, FinalAnswerResult)- assert result.answer == {"answer": "my answer", "score": 42}+ assert isinstance(result, FinalOutput)+ assert result.output == {"answer": "my answer", "score": 42}def test_submit_wrong_arg_count():