DSPy: The framework for programming—not prompting—language models
Python
Refactor ProgramOfThought and add unit tests (#7878)
TomuHirata0bb3fd4c31f3
diff --git a/dspy/predict/program_of_thought.py b/dspy/predict/program_of_thought.pyindex a6d6a8e42..421486116 100644--- a/dspy/predict/program_of_thought.py+++ b/dspy/predict/program_of_thought.py@@ -1,14 +1,37 @@+import loggingimport re+from typing import Union, Typeimport dspy-from dspy.signatures.signature import ensure_signature+from dspy.signatures.signature import ensure_signature, Signaturefrom dspy.primitives.program import Modulefrom dspy.primitives.python_interpreter import PythonInterpreter+logger = logging.getLogger(__name__)class ProgramOfThought(Module):- def __init__(self, signature, max_iters=3):+ """+ A DSPy module that runs Python programs to solve a problem.+ This module reuires deno to be installed. Please install deno following https://docs.deno.com/runtime/getting_started/installation/++ Example:+ ```+ import dspy++ lm = dspy.LM('openai/gpt-4o-mini')+ dspy.configure(lm=lm)+ pot = dspy.ProgramOfThought("question -> answer")+ pot(question="what is 1+1?")+ ```+ """++ def __init__(self, signature: Union[str, Type[Signature]], max_iters=3):+ """+ Args:+ signature: The signature of the module.+ max_iters: The maximum number of iterations to retry code generation and execution.+ """super().__init__()self.signature = signature = ensure_signature(signature)self.max_iters = max_iters@@ -56,6 +79,10 @@ class ProgramOfThought(Module):self._generate_instruction("answer"),),)+ # Currently, the interpreter class checks the deno availability at execution time.+ # We may consider checking it at the initialization time for better instruction.+ self.interpreter = PythonInterpreter()+def _generate_signature(self, mode):signature_dict = dict(self.input_fields)fields_for_mode = {@@ -125,7 +152,7 @@ class ProgramOfThought(Module):return "\n".join(instr)- def parse_code(self, code_data):+ def _parse_code(self, code_data):code = (code_data.get("generated_code", "").split("---", 1)[0].split("\n\n\n", 1)[0])@@ -148,35 +175,42 @@ class ProgramOfThought(Module):)return code_block, None- def execute_code(self, code):+ def _execute_code(self, code):+ """+ Execute the code using PythonInterpreter and return the output or error.+ """if not code:- return code, None, "Error: Empty code before execution."- interpreter = PythonInterpreter()+ return None, "Error: Empty code before execution."+try:- output = str(interpreter.execute(code))- return code, output, None+ output = str(self.interpreter.execute(code))+ return output, Noneexcept Exception as e:- return code, None, str(e)+ return None, str(e)+def forward(self, **kwargs):input_kwargs = {field_name: kwargs[field_name] for field_name in self.input_fields}code_data = self.code_generate(**input_kwargs)- parsed_code, error = self.parse_code(code_data)- # FIXME: Don't try to execute the code if it didn't parse- code, output, error = self.execute_code(parsed_code)- hop = 0- while hop < self.max_iters and error:- print("Error in code execution")+ output = None+ code, error = self._parse_code(code_data)+ if not error:+ output, error = self._execute_code(code)+ hop = 1+ # Retying code generation and execution until no error or reach max_iters+ while error is not None:+ logger.error(f"Error in code execution: {error}")+ if hop == self.max_iters:+ self.interpreter.shutdown()+ raise RuntimeError(f"Max hops reached. Failed to run ProgramOfThought: {error}")input_kwargs.update({"previous_code": code, "error": error})code_data = self.code_regenerate(**input_kwargs)- parsed_code, error = self.parse_code(code_data)- # FIXME: Don't try to execute the code if it didn't parse- code, output, error = self.execute_code(parsed_code)+ code, error = self._parse_code(code_data)+ if not error:+ output, error = self._execute_code(code)hop += 1- if hop == self.max_iters:- print("Max hops reached. Error persists.")- return Noneinput_kwargs.update({"final_generated_code": code, "code_output": output})answer_gen_result = self.generate_answer(**input_kwargs)- return answer_gen_result\ No newline at end of file+ self.interpreter.shutdown()+ return answer_gen_resultdiff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex dcf91c789..f3332f0f9 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -117,14 +117,14 @@ class PythonInterpreter:# If not valid JSON, just return raw textresult = {"output": output_line}- # If we have an error, handle SyntaxError vs. other error+ # If we have an error, determine if it's a SyntaxError or other error using error.errorType.if "error" in result:error_msg = result["error"]- error_type = result.get("errorType", "")+ error_type = result.get("errorType", "Sandbox Error")if error_type == "SyntaxError":- raise SyntaxError(error_msg)+ raise SyntaxError(f"Invalid Python syntax. message: {error_msg}")else:- raise InterpreterError(f"Sandbox Error: {error_msg}")+ raise InterpreterError(f"{error_type}: {error_msg}")# If there's no error, return the "output" fieldreturn result.get("output", None)diff --git a/dspy/primitives/runner.js b/dspy/primitives/runner.jsindex 642141078..644c6820c 100644--- a/dspy/primitives/runner.js+++ b/dspy/primitives/runner.js@@ -86,7 +86,9 @@ sys.stderr = old_stderrconsole.log(JSON.stringify({ output }));} catch (error) {// We have an error => check if it's a SyntaxError or something else- const errorType = error.name || "Error";+ // The Python error class name is stored in error.type: https://pyodide.org/en/stable/usage/api/js-api.html#pyodide.ffi.PythonError+ const errorType = error.type || "Error";+ // error.message is mostly blank.const errorMessage = (error.message || "").trim();console.log(JSON.stringify({error: errorMessage,diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.pyindex b03b7356c..aeef3aae1 100644--- a/tests/predict/test_program_of_thought.py+++ b/tests/predict/test_program_of_thought.py@@ -1,13 +1,20 @@+from unittest.mock import patch+import pytest+import shutil+import dspyfrom dspy import ProgramOfThought, Signaturefrom dspy.utils import DummyLM+# This test suite requires deno to be installed. Please install deno following https://docs.deno.com/runtime/getting_started/installation/+is_deno_available = shutil.which("deno") is not Noneclass BasicQA(Signature):question = dspy.InputField()answer = dspy.OutputField(desc="often between 1 and 5 words")[email protected](not is_deno_available, reason="Deno is not installed or not in PATH")def test_pot_code_generation():lm = DummyLM([@@ -19,9 +26,11 @@ def test_pot_code_generation():pot = ProgramOfThought(BasicQA)res = pot(question="What is 1+1?")assert res.answer == "2"+ assert pot.interpreter.deno_process is None-def test_pot_code_generation_with_error():[email protected](not is_deno_available, reason="Deno is not installed or not in PATH")+def test_pot_code_generation_with_one_error():lm = DummyLM([{"reasoning": "Reason_A", "generated_code": "```python\nresult = 1+0/0\n```"},@@ -34,3 +43,35 @@ def test_pot_code_generation_with_error():pot = ProgramOfThought(BasicQA)res = pot(question="What is 1+1?")assert res.answer == "2"+ assert pot.interpreter.deno_process is None++[email protected](not is_deno_available, reason="Deno is not installed or not in PATH")+def test_pot_code_generation_persistent_errors():+ max_iters = 3+ lm = DummyLM(+ [+ {"reasoning": "Reason_A", "generated_code": "```python\nresult = 1+0/0\n```"},+ ] * max_iters+ )+ dspy.settings.configure(lm=lm)++ pot = ProgramOfThought(BasicQA, max_iters=max_iters)+ with pytest.raises(RuntimeError, match="Max hops reached. Failed to run ProgramOfThought: ZeroDivisionError:"):+ pot(question="What is 1+1?")+ assert pot.interpreter.deno_process is None+++def test_pot_code_parse_error():+ max_iters = 3+ lm = DummyLM(+ [+ {"reasoning": "Reason_A", "generated_code": "```python\ninvalid=python=code\n```"},+ ] * max_iters+ )+ dspy.settings.configure(lm=lm)++ pot = ProgramOfThought(BasicQA, max_iters=max_iters)+ with patch("dspy.predict.program_of_thought.ProgramOfThought._execute_code") as mock_execute_code, pytest.raises(RuntimeError, match="Max hops reached. Failed to run ProgramOfThought: Error: Code format is not correct."):+ pot(question="What is 1+1?")+ mock_execute_code.assert_not_called()diff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.pyindex 55e83f2d6..593cf63fa 100644--- a/tests/primitives/test_python_interpreter.py+++ b/tests/primitives/test_python_interpreter.py@@ -1,4 +1,10 @@-from dspy.primitives.python_interpreter import PythonInterpreter+import shutil+import pytest+from dspy.primitives.python_interpreter import PythonInterpreter, InterpreterError++# This test suite requires deno to be installed. Please install deno following https://docs.deno.com/runtime/getting_started/installation/+if shutil.which("deno") is None:+ pytest.skip(reason="Deno is not installed or not in PATH")def test_execute_simple_code():interpreter = PythonInterpreter()@@ -16,4 +22,16 @@ def test_user_variable_definitions():interpreter = PythonInterpreter()code = "result = number + 1\nresult"result = interpreter.execute(code, variables={'number': 4})- assert result == 5, "User variable assignment should work"\ No newline at end of file+ assert result == 5, "User variable assignment should work"++def test_failure_syntax_error():+ interpreter = PythonInterpreter()+ code = "+++"+ with pytest.raises(SyntaxError, match="Invalid Python syntax"):+ interpreter.execute(code)++def test_failure_zero_division():+ interpreter = PythonInterpreter()+ code = "1+0/0"+ with pytest.raises(InterpreterError, match="ZeroDivisionError"):+ interpreter.execute(code)