DSPy: The framework for programming—not prompting—language models
Python
fix(rlm): protect RLM-owned namespaces (#10020)
Isaac Miller74194bfab3ec
diff --git a/dspy/predict/rlm.py b/dspy/predict/rlm.pyindex 0cc2536b0..09ec51f38 100644--- a/dspy/predict/rlm.py+++ b/dspy/predict/rlm.py@@ -14,6 +14,7 @@ import base64import contextvarsimport functoolsimport inspect+import keywordimport loggingimport threadingfrom concurrent.futures import ThreadPoolExecutor, as_completed@@ -172,7 +173,7 @@ class RLM(Module):self.sub_lm = sub_lmself._interpreter_factory = interpreter_factoryself._user_tools = self._normalize_tools(tools)- self._validate_tools(self._user_tools)+ self._validate_namespace(self._user_tools)# Build the action and extract signaturesaction_sig, extract_sig = self._build_signatures()@@ -183,8 +184,9 @@ class RLM(Module):# Tool Creation and Validation# =========================================================================- # Reserved tool names that conflict with built-in sandbox functions- _RESERVED_TOOL_NAMES = frozenset({"llm_query", "llm_query_batched", "SUBMIT", "print"})+ # Names owned by RLM rather than the user-provided signature or tools.+ _RESERVED_SANDBOX_NAMES = frozenset({"llm_query", "llm_query_batched", "SUBMIT", "print"})+ _RESERVED_RESULT_NAMES = frozenset({"trajectory", "final_reasoning"})def _normalize_tools(self, tools: list[Callable] | None) -> dict[str, Tool]:"""Normalize tools list to a dict of Tool objects keyed by name."""@@ -205,18 +207,35 @@ class RLM(Module):raise TypeError(f"Tool {func!r} must be callable, got {type(func).__name__}")return Tool(func)- # List of callables/Tools -> normalize to Tool objects- tool_list = [to_tool(t) for t in tools]- return {tool.name: tool for tool in tool_list}+ normalized = {}+ for value in tools:+ tool = to_tool(value)+ if tool.name in normalized:+ raise ValueError(f"Duplicate tool name '{tool.name}'")+ normalized[tool.name] = tool+ return normalized- def _validate_tools(self, tools: dict[str, Tool]) -> None:- """Validate user-provided tools have valid names."""+ def _validate_namespace(self, tools: dict[str, Tool]) -> None:+ """Validate names owned by the RLM result and sandbox APIs."""for name in tools:- if not name.isidentifier():- raise ValueError(f"Invalid tool name '{name}': must be a valid Python identifier")- if name in self._RESERVED_TOOL_NAMES:+ if not name.isidentifier() or keyword.iskeyword(name):+ raise ValueError(f"Invalid tool name '{name}': must be a valid Python identifier and not a keyword")+ if name in self._RESERVED_SANDBOX_NAMES:raise ValueError(f"Tool name '{name}' conflicts with built-in sandbox function")+ input_names = set(self.signature.input_fields)+ reserved_inputs = sorted(input_names & self._RESERVED_SANDBOX_NAMES)+ if reserved_inputs:+ raise ValueError(f"Input fields conflict with built-in sandbox functions: {reserved_inputs}")++ tool_inputs = sorted(input_names & tools.keys())+ if tool_inputs:+ raise ValueError(f"Input fields conflict with user tools: {tool_inputs}")++ reserved_outputs = sorted(set(self.signature.output_fields) & self._RESERVED_RESULT_NAMES)+ if reserved_outputs:+ raise ValueError(f"Output fields conflict with RLM result metadata: {reserved_outputs}")+def _format_tool_docs(self, tools: dict[str, Tool]) -> str:"""Format user-provided tools for inclusion in instructions."""if not tools:@@ -398,12 +417,17 @@ class RLM(Module):return outputdef _validate_inputs(self, input_args: dict[str, Any]) -> None:- """Raise ValueError if required input fields are missing."""+ """Validate call-time arguments against the signature's input namespace."""if "interpreter" in input_args and "interpreter" not in self.signature.input_fields:raise TypeError("To use a caller-owned interpreter, pass it as the first positional argument when calling the module.")- missing = set(self.signature.input_fields.keys()) - set(input_args.keys())+ input_names = set(self.signature.input_fields)+ unexpected = set(input_args) - input_names+ if unexpected:+ raise ValueError(f"Unexpected inputs not declared in the signature: {sorted(unexpected)}")++ missing = input_names - set(input_args)if missing:raise ValueError(f"Missing required inputs: {sorted(missing)}")diff --git a/tests/predict/test_rlm.py b/tests/predict/test_rlm.pyindex f3bbe5e58..8aeccf985 100644--- a/tests/predict/test_rlm.py+++ b/tests/predict/test_rlm.py@@ -160,7 +160,15 @@ class TestRLMInitialization:with pytest.raises(ValueError, match="must be a valid Python identifier"):RLM("context -> answer", tools=[tool])- @pytest.mark.parametrize("tool_name", ["llm_query", "SUBMIT", "print"])+ def test_tool_validation_rejects_python_keyword(self):+ def my_tool() -> str:+ return "result"++ tool = Tool(my_tool, name="for")+ with pytest.raises(ValueError, match="not a keyword"):+ RLM("context -> answer", tools=[tool])++ @pytest.mark.parametrize("tool_name", ["llm_query", "llm_query_batched", "SUBMIT", "print"])def test_tool_validation_reserved_names(self, tool_name):"""Test RLM rejects tool names that conflict with built-in functions."""def my_tool() -> str:@@ -184,6 +192,33 @@ class TestRLMInitialization:with pytest.raises(TypeError, match="tools must be a list, not a dict"):RLM("context -> answer", tools={"my_tool": my_tool})+ def test_duplicate_tool_names_rejected(self):+ def first() -> str:+ return "first"++ def second() -> str:+ return "second"++ with pytest.raises(ValueError, match="Duplicate tool name 'lookup'"):+ RLM("context -> answer", tools=[Tool(first, name="lookup"), Tool(second, name="lookup")])++ @pytest.mark.parametrize("input_name", ["llm_query", "llm_query_batched", "SUBMIT", "print"])+ def test_input_names_cannot_shadow_sandbox_functions(self, input_name):+ with pytest.raises(ValueError, match="Input fields conflict with built-in sandbox functions"):+ RLM(f"{input_name} -> answer")++ def test_input_name_cannot_shadow_user_tool(self):+ def lookup() -> str:+ return "result"++ with pytest.raises(ValueError, match="Input fields conflict with user tools: \\['lookup'\\]"):+ RLM("lookup -> answer", tools=[lookup])++ @pytest.mark.parametrize("output_name", ["trajectory", "final_reasoning"])+ def test_output_names_cannot_shadow_result_metadata(self, output_name):+ with pytest.raises(ValueError, match=f"Output fields conflict with RLM result metadata: \\['{output_name}'\\]"):+ RLM(f"context -> {output_name}")+def test_optional_parameters(self):"""Test RLM optional parameters and their defaults."""import dspy@@ -276,6 +311,21 @@ class TestRLMInitialization:with pytest.raises(TypeError, match="Sub-LM response must contain text, got NoneType"):tools["llm_query"]("test prompt")+ @pytest.mark.parametrize("unexpected_name", ["SUBMIT", "lookup", "tools"])+ def test_forward_rejects_undeclared_inputs_before_interpreter_execution(self, unexpected_name):+ def lookup() -> str:+ return "tool result"++ factory = MockInterpreterFactory(responses=[FinalOutput({"answer": "done"})])+ rlm = RLM("context -> answer", tools=[lookup], interpreter_factory=factory)+ rlm.generate_action = make_mock_predictor([+ {"reasoning": "Return answer", "code": 'SUBMIT("done")'},+ ])++ with pytest.raises(ValueError, match=f"Unexpected inputs not declared in the signature: \\['{unexpected_name}'\\]"):+ rlm(context="some context", **{unexpected_name: "shadowed value"})++ assert factory.instances == []def test_batched_query_errors_have_clear_markers(self):"""Test that errors in llm_query_batched are prefixed with [ERROR]."""@@ -1066,6 +1116,19 @@ print(json.dumps(data))class TestRLMAsyncMock:"""Unit tests for RLM aforward() using MockInterpreter (no Deno required)."""+ @pytest.mark.asyncio+ async def test_aforward_rejects_undeclared_inputs_before_interpreter_execution(self):+ factory = MockInterpreterFactory(responses=[FinalOutput({"answer": "done"})])+ rlm = RLM("context -> answer", interpreter_factory=factory)+ rlm.generate_action = make_mock_predictor([+ {"reasoning": "Return answer", "code": 'SUBMIT("done")'},+ ])++ with pytest.raises(ValueError, match="Unexpected inputs not declared in the signature: \\['SUBMIT'\\]"):+ await rlm.acall(context="some context", SUBMIT="shadowed value")++ assert factory.instances == []+@pytest.mark.asyncioasync def test_aforward_basic(self):"""Test aforward() returns Prediction with expected output (MockInterpreter)."""