DSPy: The framework for programming—not prompting—language models
Python
Update dspy.RLM to improve reliability and avoid pydantic warnings (#9210)
Omar Khattab19c995dc5873
diff --git a/dspy/predict/program_of_thought.py b/dspy/predict/program_of_thought.pyindex a854baccc..44f9d34e1 100644--- a/dspy/predict/program_of_thought.py+++ b/dspy/predict/program_of_thought.py@@ -118,8 +118,8 @@ class ProgramOfThought(Module):instr = [f"You will be given {mode_inputs} and you will respond with {mode_outputs}.",f"Generating executable Python code that programmatically computes the correct {mode_outputs}.",- "After you're done with the computation and think you have the answer, make sure to provide your answer by calling the preloaded function `FINAL()`.",- f'You should structure your answer in a dict object, like {{"field_a": answer_a, ...}}, evaluates to the correct value mapping for {final_outputs}.',+ "After you're done with the computation and think you have the final output, make sure to submit your output by calling the preloaded function `SUBMIT()`.",+ f'You must structure your output in a dict, like {{"field_a": value_a, ...}}, with the correct value mapping for the field(s): {final_outputs}.',]elif mode == "regenerate":instr = [diff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.pyindex 0517a43dc..7e27d1ef7 100644--- a/dspy/predict/rlm.py+++ b/dspy/predict/rlm.py@@ -48,8 +48,7 @@ Available:- `llm_query(prompt)` - query a sub-LLM (~500K char capacity) for semantic analysis- `llm_query_batched(prompts)` - query multiple prompts concurrently (much faster for multiple queries)- `print()` - ALWAYS print to see results-- `FINAL({final_output_names})` - submit final answer when done-- `FINAL_VAR({final_var_output_names})` - submit final answer using variable names+- `SUBMIT({final_output_names})` - submit final output when done- Standard libraries: re, json, collections, math, etc.IMPORTANT: This is ITERATIVE. Each code block you write will execute, you'll see the output, then you decide what to do next. Do NOT try to solve everything in one step.@@ -58,8 +57,10 @@ IMPORTANT: This is ITERATIVE. Each code block you write will execute, you'll see2. ITERATE - Write small code snippets, observe outputs, then decide next steps. State persists between iterations.3. VERIFY BEFORE SUBMITTING - If results seem wrong (zeros, empty, unexpected), reconsider your approach.4. USE llm_query FOR SEMANTICS - String matching finds WHERE things are; llm_query understands WHAT things mean.+5. MINIMIZE RETYPING (INPUTS & OUTPUTS) - When values are long, precise, or error-prone (IDs, numbers, code, quotes), re-access them via variables and parse/compute in code instead of retyping. Use small, targeted prints to sanity-check, but avoid manual copying when variables can carry the exact value.+6. SUBMIT ONLY AFTER SEEING OUTPUTS - SUBMIT ends the current run immediately. If you need to inspect printed output, run it in one step, review the result, then call SUBMIT in a later step.-You have max {max_llm_calls} sub-LLM calls. When done, call FINAL() or FINAL_VAR() with your answer."""+You have max {max_llm_calls} sub-LLM calls. When done, call SUBMIT() with your output."""# Pattern to match markdown code fences: ```python\n...\n``` or ```\n...\n```_CODE_FENCE_PATTERN = re.compile(r"^```(?:python|py)?\s*\n(.*?)\n```\s*$", re.DOTALL)@@ -143,7 +144,7 @@ class RLM(Module):# =========================================================================# Reserved tool names that conflict with built-in sandbox functions- _RESERVED_TOOL_NAMES = frozenset({"llm_query", "llm_query_batched", "FINAL", "FINAL_VAR", "print"})+ _RESERVED_TOOL_NAMES = frozenset({"llm_query", "llm_query_batched", "SUBMIT", "print"})def _validate_tools(self, tools: dict[str, Callable]) -> None:"""Validate user-provided tools have valid names and are callable."""@@ -250,12 +251,9 @@ class RLM(Module):"""Build the action and extract signatures from templates."""inputs_str = ", ".join(f"`{n}`" for n in self.signature.input_fields)- # Simple names for FINAL() examples+ # Simple names for SUBMIT() examplesfinal_output_names = ", ".join(self.signature.output_fields.keys())- # Final output names for FINAL_VAR() examples- final_var_output_names = ", ".join(f"`{n}`" for n in self.signature.output_fields.keys())-output_fields = "\n".join(f"- {translate_field_type(n, f)}"for n, f in self.signature.output_fields.items()@@ -269,10 +267,10 @@ class RLM(Module):action_sig = (dspy.Signature({}, task_instructions + ACTION_INSTRUCTIONS_TEMPLATE.format(- inputs=inputs_str, final_output_names=final_output_names, final_var_output_names=final_var_output_names, output_fields=output_fields,+ inputs=inputs_str, final_output_names=final_output_names, output_fields=output_fields,max_llm_calls=self.max_llm_calls,) + tool_docs)- .append("variables_info", dspy.InputField(desc="Metadata about the variables available in the REPL"), type_=list[REPLVariable])+ .append("variables_info", dspy.InputField(desc="Metadata about the variables available in the REPL"), type_=str).append("repl_history", dspy.InputField(desc="Previous REPL code executions and their outputs"), type_=REPLHistory).append("iteration", dspy.InputField(desc="Current iteration number (1-indexed) out of max_iterations"), type_=str).append("reasoning", dspy.OutputField(desc="Think step-by-step: what do you know? What remains? Plan your next action."), type_=str)@@ -295,7 +293,7 @@ class RLM(Module):full_extract_instructions,)extract_sig = extract_sig.prepend("repl_history", dspy.InputField(desc="Your REPL interactions so far"), type_=REPLHistory)- extract_sig = extract_sig.prepend("variables_info", dspy.InputField(desc="Metadata about the variables available in the REPL"), type_=list[REPLVariable])+ extract_sig = extract_sig.prepend("variables_info", dspy.InputField(desc="Metadata about the variables available in the REPL"), type_=str)return action_sig, extract_sig@@ -389,10 +387,11 @@ class RLM(Module):output_field_names: list[str],) -> Prediction:"""Use extract module to get final answer when max iterations reached."""- logger.warning("RLM reached max iterations, using extract to get final answer")+ logger.warning("RLM reached max iterations, using extract to get final output")+ variables_info = [variable.format() for variable in variables]extract_pred = self.extract(- variables_info=variables,+ variables_info=variables_info,repl_history=history,)@@ -417,7 +416,7 @@ class RLM(Module):# Validate all required output fields are presentmissing = set(output_field_names) - set(raw_answer.keys())if missing:- return None, f"[Error] Missing output fields: {sorted(missing)}. Use FINAL({', '.join(output_field_names)})"+ return None, f"[Error] Missing output fields: {sorted(missing)}. Use SUBMIT({', '.join(output_field_names)})"# Parse and validate each output fieldparsed_outputs = {}@@ -498,8 +497,9 @@ class RLM(Module):output_field_names: list[str],) -> Prediction | REPLHistory:"""Execute one iteration. Returns Prediction if done, else updated REPLHistory."""+ variables_info = [variable.format() for variable in variables]action = self.generate_action(- variables_info=variables,+ variables_info=variables_info,repl_history=history,iteration=f"{iteration + 1}/{self.max_iterations}",)@@ -560,10 +560,11 @@ class RLM(Module):output_field_names: list[str],) -> Prediction:"""Async version: Use extract module when max iterations reached."""- logger.warning("RLM reached max iterations, using extract to get final answer")+ logger.warning("RLM reached max iterations, using extract to get final output")+ variables_info = [variable.format() for variable in variables]extract_pred = await self.extract.acall(- variables_info=variables,+ variables_info=variables_info,repl_history=history,)@@ -583,8 +584,9 @@ class RLM(Module):output_field_names: list[str],) -> Prediction | REPLHistory:"""Async version: Execute one iteration."""+ variables_info = [variable.format() for variable in variables]pred = await self.generate_action.acall(- variables_info=variables,+ variables_info=variables_info,repl_history=history,iteration=f"{iteration + 1}/{self.max_iterations}",)diff --git a/dspy/primitives/code_interpreter.py b/dspy/primitives/code_interpreter.pyindex 9565faf69..3838074fe 100644--- a/dspy/primitives/code_interpreter.py+++ b/dspy/primitives/code_interpreter.py@@ -9,7 +9,7 @@ code-executing modules to work with different interpreter implementations:from typing import Any, Callable, Protocol, runtime_checkable-# Types that can be used directly in Python function signatures for FINAL()+# Types that can be used directly in Python function signatures for SUBMIT()SIMPLE_TYPES = (str, int, float, bool, list, dict, type(None))@@ -22,7 +22,7 @@ class CodeInterpreterError(RuntimeError):class FinalAnswerResult:- """Returned by interpreter.execute() when FINAL() or FINAL_VAR() is called.+ """Returned by interpreter.execute() when SUBMIT() is called.This signals that the code execution loop should terminate and returnthe contained answer to the caller.@@ -107,7 +107,7 @@ class CodeInterpreter(Protocol):Returns:One of:- - FinalAnswerResult: If FINAL() or FINAL_VAR() was called in code+ - FinalAnswerResult: If SUBMIT() was called in code- str: Captured stdout from print() statements- list: Multiple output lines- None: If no output was produceddiff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex 4782f92af..b492058db 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -71,7 +71,7 @@ class PythonInterpreter:tools: Dictionary mapping tool names to callable functions.Each function should accept keyword arguments and return a string.Tools are callable directly from sandbox code by name.- output_fields: List of output field definitions for typed FINAL signature.+ output_fields: List of output field definitions for typed SUBMIT signature.Each dict should have 'name' and optionally 'type' keys."""if isinstance(deno_command, dict):diff --git a/dspy/primitives/runner.js b/dspy/primitives/runner.jsindex 22be7e8a7..28d9f5a25 100644--- a/dspy/primitives/runner.js+++ b/dspy/primitives/runner.js@@ -8,7 +8,7 @@ import { readLines } from "https://deno.land/[email protected]/io/mod.ts";// =============================================================================// Setup code run before each user code execution.-// Captures stdout, defines FINAL/FINAL_VAR for early termination, and+// Captures stdout, defines SUBMIT for early termination, and// provides a helper to extract exception args across the JS/Python boundary.const PYTHON_SETUP_CODE = `import sys, io, json@@ -23,17 +23,11 @@ class FinalAnswer(BaseException):# Control-flow exception to signal completion (like StopIteration)pass-# Default FINAL/FINAL_VAR for single-output signatures (e.g., Program of Thought).+# Default SUBMIT for single-output signatures (e.g., Program of Thought).# Only define if not already registered with typed signatures.-if 'FINAL' not in dir():- def FINAL(answer):+if 'SUBMIT' not in dir():+ def SUBMIT(answer):raise FinalAnswer({"answer": answer})--if 'FINAL_VAR' not in dir():- def FINAL_VAR(var_name):- if var_name in globals():- raise FinalAnswer({"answer": globals()[var_name]})- raise NameError(f"Variable '{var_name}' not found")`;// Generate a tool wrapper function with typed signature.@@ -78,13 +72,13 @@ def ${toolName}(${signature}):`;};-// Generate FINAL function with output field signature.+// Generate SUBMIT function with output field signature.// Outputs is an array of {name, type?} objects.-const makeFinalWrapper = (outputs) => {+const makeSubmitWrapper = (outputs) => {if (!outputs || outputs.length === 0) {- // Fallback to single-arg FINAL if no outputs defined+ // Fallback to single-arg SUBMIT if no outputs definedreturn `-def FINAL(answer):+def SUBMIT(answer):raise FinalAnswer({"answer": answer})`;}@@ -97,51 +91,11 @@ def FINAL(answer):const dictParts = outputs.map(o => `"${o.name}": ${o.name}`);return `-def FINAL(${sigParts.join(', ')}):+def SUBMIT(${sigParts.join(', ')}):raise FinalAnswer({${dictParts.join(', ')}})`;};-// Generate FINAL_VAR function with positional args mapped to output fields.-// Usage: FINAL_VAR("var1", "var2") maps to output fields in order.-const makeFinalVarWrapper = (outputs) => {- if (!outputs || outputs.length === 0) {- // Single-output fallback: FINAL_VAR("var_name")- return `-def FINAL_VAR(var_name):- if var_name not in globals():- raise NameError(f"Variable '{var_name}' not found")- raise FinalAnswer({"answer": globals()[var_name]})-`;- }-- // Multi-output: positional args mapped to fields in order- const fieldNames = outputs.map(o => `"${o.name}"`);- const expectedCount = outputs.length;- const fieldList = fieldNames.join(', ');-- return `-def FINAL_VAR(*var_names):- """Pass variable names positionally, mapped to output fields in order.-- Output fields: ${outputs.map(o => o.name).join(', ')}- Usage: FINAL_VAR(${outputs.map(o => `"${o.name}_var"`).join(', ')})- """- _output_fields = [${fieldList}]- if len(var_names) != ${expectedCount}:- raise ValueError(- f"FINAL_VAR expects ${expectedCount} variable names for output fields {_output_fields}, "- f"got {len(var_names)}"- )- _result = {}- for field_name, var_name in zip(_output_fields, var_names):- if var_name not in globals():- raise NameError(f"Variable '{var_name}' not found for output field '{field_name}'")- _result[field_name] = globals()[var_name]- raise FinalAnswer(_result)-`;-};-// Global handler to prevent uncaught promise rejections from crashing Deno// These can occur during async Python <-> JS interopglobalThis.addEventListener("unhandledrejection", (event) => {@@ -287,10 +241,9 @@ while (true) {}}- // Register FINAL/FINAL_VAR with output signature+ // Register SUBMIT with output signatureif (input.register_outputs) {- pyodide.runPython(makeFinalWrapper(input.register_outputs));- pyodide.runPython(makeFinalVarWrapper(input.register_outputs));+ pyodide.runPython(makeSubmitWrapper(input.register_outputs));}console.log(JSON.stringify({@@ -332,4 +285,4 @@ while (true) {}console.log(JSON.stringify({ error: errorMessage, errorArgs, errorType }));}-}\ No newline at end of file+}diff --git a/tests/mock_interpreter.py b/tests/mock_interpreter.pyindex 8b26ff581..e36ceecf1 100644--- a/tests/mock_interpreter.py+++ b/tests/mock_interpreter.py@@ -4,7 +4,7 @@ Mock interpreter for testing RLM and other code-executing modules.This interpreter doesn't actually execute code - it returns scripted responsesor uses a custom function to generate responses. Useful for:- Unit testing without Deno/Pyodide dependencies-- Testing specific execution paths (errors, FINAL, etc.)+- Testing specific execution paths (errors, SUBMIT, etc.)- Recording what code was submitted for execution"""@@ -28,11 +28,11 @@ class MockInterpreter:FinalAnswerResult("42"),])result1 = mock.execute("print(len(context))") # Returns "data explored"- result2 = mock.execute("FINAL('42')") # Returns FinalAnswerResult("42")+ result2 = mock.execute("SUBMIT('42')") # Returns FinalAnswerResult("42")# Use custom execution functiondef custom_exec(code, variables):- if "FINAL" in code:+ if "SUBMIT" in code:return FinalAnswerResult("done")return f"executed: {code[:20]}..."diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.pyindex 04ab78ac8..72cf95502 100644--- a/tests/predict/test_program_of_thought.py+++ b/tests/predict/test_program_of_thought.py@@ -22,7 +22,7 @@ def test_pot_code_generation():[{"reasoning": "Reason_A",- "generated_code": "```python\nresult = 1+1\nFINAL({'answer': result})\n```",+ "generated_code": "```python\nresult = 1+1\nSUBMIT({'answer': result})\n```",},{"reasoning": "Reason_B", "answer": "2"},]@@ -62,7 +62,7 @@ def test_pot_support_multiple_fields():[{"reasoning": "Reason_A",- "generated_code": "```python\nmaximum = 6\nminimum = 2\nFINAL({'maximum': maximum, 'minimum': minimum})\n```",+ "generated_code": "```python\nmaximum = 6\nminimum = 2\nSUBMIT({'maximum': maximum, 'minimum': minimum})\n```",},{"reasoning": "Reason_B", "maximum": "6", "minimum": "2"},]@@ -81,11 +81,11 @@ def test_pot_code_generation_with_one_error():[{"reasoning": "Reason_A",- "generated_code": "```python\nresult = 1+0/0\nFINAL({'answer': result})\n```",+ "generated_code": "```python\nresult = 1+0/0\nSUBMIT({'answer': result})\n```",},{"reasoning": "Reason_B",- "generated_code": "```python\nresult = 1+1\nFINAL({'answer': result})\n```",+ "generated_code": "```python\nresult = 1+1\nSUBMIT({'answer': result})\n```",},{"reasoning": "Reason_C", "answer": "2"},]@@ -104,7 +104,7 @@ def test_pot_code_generation_persistent_errors():[{"reasoning": "Reason_A",- "generated_code": "```python\nresult = 1+0/0\nFINAL({'answer': result})\n```",+ "generated_code": "```python\nresult = 1+0/0\nSUBMIT({'answer': result})\n```",},]* max_itersdiff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.pyindex dd65e2691..fbbab742e 100644--- a/tests/predict/test_rlm.py+++ b/tests/predict/test_rlm.py@@ -93,7 +93,7 @@ class TestMockInterpreter:"""Test that MockInterpreter can return FinalAnswerResult."""mock = MockInterpreter(responses=["exploring", FinalAnswerResult("42")])assert mock.execute("print(len(data))") == "exploring"- result = mock.execute("FINAL('42')")+ result = mock.execute("SUBMIT('42')")assert isinstance(result, FinalAnswerResult)assert result.answer == "42"@@ -156,7 +156,7 @@ class TestRLMInitialization:with pytest.raises(ValueError, match="must be a valid Python identifier"):RLM("context -> answer", tools={tool_name: my_tool})- @pytest.mark.parametrize("tool_name", ["llm_query", "FINAL", "print"])+ @pytest.mark.parametrize("tool_name", ["llm_query", "SUBMIT", "print"])def test_tool_validation_reserved_names(self, tool_name):"""Test RLM rejects tool names that conflict with built-in functions."""def my_tool() -> str:@@ -443,7 +443,7 @@ class TestRLMCallMethod:mock = MockInterpreter(responses=[FinalAnswerResult({"answer": "42"})])rlm = RLM("query -> answer", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return answer", "code": 'FINAL("42")'},+ {"reasoning": "Return answer", "code": 'SUBMIT("42")'},])result = rlm(query="What is the answer?")@@ -491,7 +491,7 @@ class TestRLMToolExceptions:rlm = RLM("query -> answer", max_iterations=5, interpreter=mock, tools={"failing_tool": failing_tool})rlm.generate_action = make_mock_predictor([{"reasoning": "Call tool", "code": "failing_tool()"},- {"reasoning": "Recover", "code": 'FINAL("recovered")'},+ {"reasoning": "Recover", "code": 'SUBMIT("recovered")'},])result = rlm.forward(query="test")@@ -516,7 +516,7 @@ class TestRLMDynamicSignature:instructions = action_sig.instructionsassert "llm_query" in instructionsassert "llm_query_batched" in instructions- assert "FINAL" in instructions+ assert "SUBMIT" in instructionsassert "`document`" in instructionsassert "`question`" in instructionsassert "`summary`" in instructions@@ -715,7 +715,7 @@ class TestRLMAsyncMock:mock = MockInterpreter(responses=[FinalAnswerResult({"answer": "42"})])rlm = RLM("query -> answer", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return answer", "code": 'FINAL("42")'},+ {"reasoning": "Return answer", "code": 'SUBMIT("42")'},])result = await rlm.aforward(query="What is the answer?")@@ -727,7 +727,7 @@ class TestRLMAsyncMock:mock = MockInterpreter(responses=[FinalAnswerResult({"count": 42})])rlm = RLM("query -> count: int", max_iterations=3, interpreter=mock)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return count", "code": "FINAL(42)"},+ {"reasoning": "Return count", "code": "SUBMIT(42)"},])result = await rlm.aforward(query="count items")@@ -736,7 +736,7 @@ class TestRLMAsyncMock:@pytest.mark.asyncioasync def test_aforward_multi_iteration_mock(self):- """Test aforward() handles multiple iterations before FINAL (MockInterpreter)."""+ """Test aforward() handles multiple iterations before SUBMIT (MockInterpreter)."""mock = MockInterpreter(responses=["explored data",FinalAnswerResult({"answer": "done"}),@@ -744,7 +744,7 @@ class TestRLMAsyncMock:rlm = RLM("query -> answer", max_iterations=5, interpreter=mock)rlm.generate_action = make_mock_predictor([{"reasoning": "Explore first", "code": "print('exploring')"},- {"reasoning": "Now finish", "code": 'FINAL("done")'},+ {"reasoning": "Now finish", "code": 'SUBMIT("done")'},])result = await rlm.aforward(query="test")@@ -755,11 +755,11 @@ class TestRLMTypeCoercionMock:"""Unit tests for RLM type coercion using MockInterpreter (no Deno required)."""@pytest.mark.parametrize("output_field,output_type,final_value,code,expected", [- ("count", "int", 42, "FINAL(42)", 42),- ("score", "float", 3.14, "FINAL(3.14)", 3.14),- ("valid", "bool", True, "FINAL(True)", True),- ("numbers", "list[int]", [1, 2, 3], "FINAL([1, 2, 3])", [1, 2, 3]),- ("answer", "Literal['yes', 'no']", "yes", 'FINAL("yes")', "yes"),+ ("count", "int", 42, "SUBMIT(42)", 42),+ ("score", "float", 3.14, "SUBMIT(3.14)", 3.14),+ ("valid", "bool", True, "SUBMIT(True)", True),+ ("numbers", "list[int]", [1, 2, 3], "SUBMIT([1, 2, 3])", [1, 2, 3]),+ ("answer", "Literal['yes', 'no']", "yes", 'SUBMIT("yes")', "yes"),])def test_type_coercion(self, output_field, output_type, final_value, code, expected):"""Test RLM type coercion for various types (MockInterpreter)."""@@ -780,8 +780,8 @@ class TestRLMTypeCoercionMock:])rlm = RLM("query -> answer: Literal['yes', 'no']", max_iterations=5, interpreter=mock)rlm.generate_action = make_mock_predictor([- {"reasoning": "Try maybe", "code": 'FINAL("maybe")'},- {"reasoning": "Try yes", "code": 'FINAL("yes")'},+ {"reasoning": "Try maybe", "code": 'SUBMIT("maybe")'},+ {"reasoning": "Try yes", "code": 'SUBMIT("yes")'},])result = rlm.forward(query="is it yes?")@@ -798,16 +798,16 @@ class TestRLMTypeCoercion:"""Tests for RLM type coercion through full forward pass with PythonInterpreter.Note: These tests let RLM create its own PythonInterpreter so it can register- typed output_fields for FINAL based on the signature.+ typed output_fields for SUBMIT based on the signature."""@pytest.mark.parametrize("output_field,output_type,code,expected,expected_type", [- ("count", "int", "FINAL(42)", 42, int),- ("score", "float", "FINAL(3.14)", 3.14, float),- ("valid", "bool", "FINAL(True)", True, bool),- ("numbers", "list[int]", "FINAL([1, 2, 3])", [1, 2, 3], list),- ("data", "dict[str, str]", 'FINAL({"key": "value"})', {"key": "value"}, dict),- ("answer", "Literal['yes', 'no']", 'FINAL("yes")', "yes", str),+ ("count", "int", "SUBMIT(42)", 42, int),+ ("score", "float", "SUBMIT(3.14)", 3.14, float),+ ("valid", "bool", "SUBMIT(True)", True, bool),+ ("numbers", "list[int]", "SUBMIT([1, 2, 3])", [1, 2, 3], list),+ ("data", "dict[str, str]", 'SUBMIT({"key": "value"})', {"key": "value"}, dict),+ ("answer", "Literal['yes', 'no']", 'SUBMIT("yes")', "yes", str),])def test_type_coercion(self, output_field, output_type, code, expected, expected_type):"""Test RLM type coercion for various types with PythonInterpreter."""@@ -820,11 +820,11 @@ class TestRLMTypeCoercion:assert getattr(result, output_field) == expectedassert isinstance(getattr(result, output_field), expected_type)- def test_final_var_extracts_typed_value(self):- """Test RLM FINAL_VAR correctly extracts typed value."""+ def test_submit_extracts_typed_value(self):+ """Test RLM SUBMIT correctly extracts typed value."""rlm = RLM("query -> count: int", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Compute and return", "code": 'result = 42\nFINAL_VAR("result")'},+ {"reasoning": "Compute and return", "code": "result = 42\nSUBMIT(result)"},])result = rlm.forward(query="count items")@@ -841,14 +841,14 @@ class TestRLMTypeCoercion:class TestRLMMultipleOutputs:"""Tests for signatures with multiple typed output fields.- Tests FINAL() and FINAL_VAR() calling patterns with multi-output signatures.+ Tests SUBMIT() calling patterns with multi-output signatures."""def test_multi_output_final_kwargs(self):- """FINAL(field1=val1, field2=val2) with keyword args."""+ """SUBMIT(field1=val1, field2=val2) with keyword args."""rlm = RLM("query -> name: str, count: int", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return both outputs", "code": 'FINAL(name="alice", count=5)'},+ {"reasoning": "Return both outputs", "code": 'SUBMIT(name="alice", count=5)'},])result = rlm.forward(query="test")@@ -857,10 +857,10 @@ class TestRLMMultipleOutputs:assert isinstance(result.count, int)def test_multi_output_final_positional(self):- """FINAL(val1, val2) with positional args mapped to field order."""+ """SUBMIT(val1, val2) with positional args mapped to field order."""rlm = RLM("query -> name: str, count: int", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return both outputs positionally", "code": 'FINAL("bob", 10)'},+ {"reasoning": "Return both outputs positionally", "code": 'SUBMIT("bob", 10)'},])result = rlm.forward(query="test")@@ -871,7 +871,7 @@ class TestRLMMultipleOutputs:"""Signature with 3+ output fields of different types."""rlm = RLM("query -> name: str, age: int, active: bool", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return all three", "code": 'FINAL(name="carol", age=30, active=True)'},+ {"reasoning": "Return all three", "code": 'SUBMIT(name="carol", age=30, active=True)'},])result = rlm.forward(query="test")@@ -880,11 +880,11 @@ class TestRLMMultipleOutputs:assert result.active is Truedef test_multi_output_final_missing_field_errors(self):- """FINAL() with missing field should return error in output."""+ """SUBMIT() with missing field should return error in output."""rlm = RLM("query -> name: str, count: int", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Missing count field", "code": 'FINAL(name="alice")'},- {"reasoning": "Now provide both", "code": 'FINAL(name="alice", count=5)'},+ {"reasoning": "Missing count field", "code": 'SUBMIT(name="alice")'},+ {"reasoning": "Now provide both", "code": 'SUBMIT(name="alice", count=5)'},])# RLM should retry after getting error for missing field@@ -892,46 +892,22 @@ class TestRLMMultipleOutputs:assert result.name == "alice"assert result.count == 5- def test_multi_output_final_var(self):- """FINAL_VAR("var1", "var2") maps variables to output fields."""+ def test_multi_output_submit_vars(self):+ """SUBMIT can pass variables directly for multiple outputs."""rlm = RLM("query -> name: str, count: int", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Use FINAL_VAR", "code": 'n = "dave"\nc = 15\nFINAL_VAR("n", "c")'},+ {"reasoning": "Use SUBMIT", "code": 'n = "dave"\nc = 15\nSUBMIT(n, c)'},])result = rlm.forward(query="test")assert result.name == "dave"assert result.count == 15- def test_multi_output_final_var_wrong_count_errors(self):- """FINAL_VAR with wrong number of args should error and retry."""- rlm = RLM("query -> name: str, count: int", max_iterations=3)- rlm.generate_action = make_mock_predictor([- {"reasoning": "Wrong arg count", "code": 'n = "eve"\nFINAL_VAR("n")'}, # Missing second arg- {"reasoning": "Now correct", "code": 'FINAL(name="eve", count=20)'},- ])-- result = rlm.forward(query="test")- assert result.name == "eve"- assert result.count == 20-- def test_multi_output_final_var_undefined_errors(self):- """FINAL_VAR with undefined variable should error and retry."""- rlm = RLM("query -> name: str, count: int", max_iterations=3)- rlm.generate_action = make_mock_predictor([- {"reasoning": "Undefined var", "code": 'n = "frank"\nFINAL_VAR("n", "undefined_var")'},- {"reasoning": "Now correct", "code": 'FINAL(name="frank", count=25)'},- ])-- result = rlm.forward(query="test")- assert result.name == "frank"- assert result.count == 25-def test_multi_output_type_coercion(self):"""Each output field is coerced to its declared type."""rlm = RLM("query -> count: int, ratio: float, flag: bool", max_iterations=3)rlm.generate_action = make_mock_predictor([- {"reasoning": "Return mixed types", "code": "FINAL(count=42, ratio=3.14, flag=True)"},+ {"reasoning": "Return mixed types", "code": "SUBMIT(count=42, ratio=3.14, flag=True)"},])result = rlm.forward(query="test")@@ -953,13 +929,13 @@ class TestRLMWithDummyLM:"""End-to-end tests using DummyLM with RLM and PythonInterpreter.Note: These tests let RLM create its own PythonInterpreter so it can register- typed output_fields for FINAL based on the signature.+ typed output_fields for SUBMIT based on the signature."""def test_simple_computation_e2e(self):"""Test full RLM pipeline: DummyLM -> RLM -> PythonInterpreter -> result."""with dummy_lm_context([- {"reasoning": "I need to compute 2 + 3", "code": "result = 2 + 3\nFINAL(result)"},+ {"reasoning": "I need to compute 2 + 3", "code": "result = 2 + 3\nSUBMIT(result)"},]):rlm = RLM("query -> answer: int", max_iterations=3)result = rlm.forward(query="What is 2 + 3?")@@ -968,10 +944,10 @@ class TestRLMWithDummyLM:assert isinstance(result.answer, int)def test_multi_turn_computation_e2e(self):- """Test RLM with multiple turns before FINAL."""+ """Test RLM with multiple turns before SUBMIT."""with dummy_lm_context([{"reasoning": "First explore the data", "code": "x = 10\nprint(f'x = {x}')"},- {"reasoning": "Now compute and return", "code": "y = x * 2\nFINAL(y)"},+ {"reasoning": "Now compute and return", "code": "y = x * 2\nSUBMIT(y)"},]):rlm = RLM("query -> answer: int", max_iterations=5)result = rlm.forward(query="Double ten")@@ -982,7 +958,7 @@ class TestRLMWithDummyLM:def test_with_input_variables_e2e(self):"""Test RLM with input variables passed to sandbox."""with dummy_lm_context([- {"reasoning": "Sum the numbers in the list", "code": "FINAL(sum(numbers))"},+ {"reasoning": "Sum the numbers in the list", "code": "SUBMIT(sum(numbers))"},]):rlm = RLM("numbers: list[int] -> total: int", max_iterations=3)result = rlm.forward(numbers=[1, 2, 3, 4, 5])@@ -995,7 +971,7 @@ class TestRLMWithDummyLM:return {"apple": "red", "banana": "yellow"}.get(key, "unknown")with dummy_lm_context([- {"reasoning": "Look up the color of apple", "code": 'color = lookup(key="apple")\nFINAL(color)'},+ {"reasoning": "Look up the color of apple", "code": 'color = lookup(key="apple")\nSUBMIT(color)'},]):rlm = RLM("fruit -> color: str", max_iterations=3, tools={"lookup": lookup})result = rlm.forward(fruit="apple")@@ -1006,7 +982,7 @@ class TestRLMWithDummyLM:async def test_aforward_simple_computation_e2e(self):"""Test aforward() full pipeline: DummyLM -> RLM -> PythonInterpreter -> result."""with dummy_lm_context([- {"reasoning": "I need to compute 2 + 3", "code": "result = 2 + 3\nFINAL(result)"},+ {"reasoning": "I need to compute 2 + 3", "code": "result = 2 + 3\nSUBMIT(result)"},]):rlm = RLM("query -> answer: int", max_iterations=3)result = await rlm.aforward(query="What is 2 + 3?")@@ -1016,10 +992,10 @@ class TestRLMWithDummyLM:@pytest.mark.asyncioasync def test_aforward_multi_turn_e2e(self):- """Test aforward() with multiple turns before FINAL."""+ """Test aforward() with multiple turns before SUBMIT."""with dummy_lm_context([{"reasoning": "First explore the data", "code": "x = 10\nprint(f'x = {x}')"},- {"reasoning": "Now compute and return", "code": "y = x * 2\nFINAL(y)"},+ {"reasoning": "Now compute and return", "code": "y = x * 2\nSUBMIT(y)"},]):rlm = RLM("query -> answer: int", max_iterations=5)result = await rlm.aforward(query="Double ten")@@ -1031,7 +1007,7 @@ class TestRLMWithDummyLM:async def test_aforward_with_input_variables_e2e(self):"""Test aforward() with input variables passed to sandbox."""with dummy_lm_context([- {"reasoning": "Sum the numbers in the list", "code": "FINAL(sum(numbers))"},+ {"reasoning": "Sum the numbers in the list", "code": "SUBMIT(sum(numbers))"},]):rlm = RLM("numbers: list[int] -> total: int", max_iterations=3)result = await rlm.aforward(numbers=[1, 2, 3, 4, 5])diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.pyindex bd72145e7..fef4965e1 100644--- a/tests/primitives/test_python_interpreter.py+++ b/tests/primitives/test_python_interpreter.py@@ -67,16 +67,16 @@ def test_exception_args():interpreter.execute(code)-def test_final_with_list():- """Test FINAL() with a list argument returns FinalAnswerResult with dict format."""+def test_submit_with_list():+ """Test SUBMIT() with a list argument returns FinalAnswerResult with dict format."""with PythonInterpreter() as interpreter:token = random.randint(1, 10**9)- code = f"FINAL(['The result is', {token}])"+ code = f"SUBMIT(['The result is', {token}])"result = interpreter(code)assert isinstance(result, FinalAnswerResult)- # FINAL now always returns a dict with "answer" key for single-output default+ # SUBMIT now always returns a dict with "answer" key for single-output defaultassert result.answer == {"answer": ["The result is", token]}def test_enable_env_vars_flag():@@ -312,11 +312,11 @@ def test_tool_default_args():# =============================================================================-# Multi-Output FINAL Tests+# Multi-Output SUBMIT Tests# =============================================================================-def test_final_with_typed_signature():- """Test FINAL with typed output signature."""+def test_submit_with_typed_signature():+ """Test SUBMIT with typed output signature."""output_fields = [{"name": "answer", "type": "str"},@@ -324,14 +324,14 @@ def test_final_with_typed_signature():]with PythonInterpreter(output_fields=output_fields) as sandbox:- result = sandbox.execute('FINAL(answer="the answer", confidence=0.95)')+ result = sandbox.execute('SUBMIT(answer="the answer", confidence=0.95)')assert isinstance(result, FinalAnswerResult)assert result.answer == {"answer": "the answer", "confidence": 0.95}-def test_final_positional_args():- """Test FINAL with positional arguments."""+def test_submit_positional_args():+ """Test SUBMIT with positional arguments."""output_fields = [{"name": "answer", "type": "str"},@@ -339,14 +339,14 @@ def test_final_positional_args():]with PythonInterpreter(output_fields=output_fields) as sandbox:- result = sandbox.execute('FINAL("the answer", 0.95)')+ result = sandbox.execute('SUBMIT("the answer", 0.95)')assert isinstance(result, FinalAnswerResult)assert result.answer == {"answer": "the answer", "confidence": 0.95}-def test_final_var_multi_output():- """Test FINAL_VAR with multiple output fields using positional args."""+def test_submit_multi_output():+ """Test SUBMIT with multiple output fields using positional args."""output_fields = [{"name": "answer", "type": "str"},@@ -354,11 +354,11 @@ def test_final_var_multi_output():]with PythonInterpreter(output_fields=output_fields) as sandbox:- # Positional args: variable names mapped to output fields in order+ # Positional args: values mapped to output fields in ordercode = """a = "my answer"s = 42-FINAL_VAR("a", "s")+SUBMIT(a, s)"""result = sandbox.execute(code)@@ -366,8 +366,8 @@ FINAL_VAR("a", "s")assert result.answer == {"answer": "my answer", "score": 42}-def test_final_var_wrong_arg_count():- """Test FINAL_VAR with wrong number of args gives clear error."""+def test_submit_wrong_arg_count():+ """Test SUBMIT with wrong number of args gives clear error."""output_fields = [{"name": "answer", "type": "str"},@@ -376,8 +376,8 @@ def test_final_var_wrong_arg_count():with PythonInterpreter(output_fields=output_fields) as sandbox:with pytest.raises(CodeInterpreterError) as exc_info:- sandbox.execute('x = 1; FINAL_VAR("x")') # Only 1 arg, expects 2- assert "expects 2 variable names" in str(exc_info.value)+ sandbox.execute("x = 1; SUBMIT(x)") # Only 1 arg, expects 2+ assert "missing 1 required positional argument" in str(exc_info.value)def test_extract_parameters():@@ -406,4 +406,3 @@ def test_extract_parameters_complex_types():# Complex types like Union are not included in type annotationassert params[0] == {"name": "items", "default": None}assert params[1] == {"name": "data", "default": None}-