DSPy: The framework for programming—not prompting—language models
Python
Initial comment revisions
Isaac Miller044c95425bcd
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.pyindex e63e9365a..a484186e0 100644--- a/dspy/adapters/chat_adapter.py+++ b/dspy/adapters/chat_adapter.py@@ -67,23 +67,9 @@ class ChatAdapter(Adapter):return fields- def format_completion(self, signature, outputs):- reconstructed = []+ def format_turn(self, signature, values, role, incomplete=False):+ return format_turn(signature, values, role, incomplete)- fields_dict = signature.output_fields-- field_name_output_map = {field: outputs[field] for field in fields_dict.keys()}-- for field, value in field_name_output_map.items():- reconstructed.append(f"[[ ## {field} ## ]]")- reconstructed.append(str(value))- reconstructed.append("") # Add an empty line for separation-- reconstructed.append("[[ ## completed ## ]]")-- result = "\n".join(reconstructed).strip()-- return resultdef format_blob(blob):if "\n" not in blob and "«" not in blob and "»" not in blob:diff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex 372daec6b..673cf905d 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -1,4 +1,3 @@-from __future__ import annotations # Used for self type hintsfrom functools import lru_cacheimport loggingimport random@@ -25,7 +24,7 @@ def warn_once(msg: str):def assert_structural_equivalency_for_predictors(predictor1: object,predictor2: object,- ) -> Optional[AssertionError]:+ ):"""Assert that the two predictors are structurally equivalent to each other.Args:diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.pyindex e66783716..78819b6ad 100644--- a/dspy/primitives/__init__.py+++ b/dspy/primitives/__init__.py@@ -3,3 +3,4 @@ from .example import *from .prediction import *from .program import *from .python_interpreter import *+from .utils import *\ No newline at end of filediff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex 2ee82f605..9f9eaf74c 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -1,4 +1,3 @@-from __future__ import annotations # Used for self type hintsfrom typing import Optionalimport magicattr@@ -7,199 +6,10 @@ from dsp.modules.lm import LMimport dspyfrom dspy.primitives.assertions import *from dspy.primitives.module import BaseModule---#--------------------------------------------------------------------------------# Templates for the user-facing strings used by this module-#---------------------------------------------------------------------------------_ERR_MSG_NUM_PREDICTORS = """Structurally equivalent programs must have the \-the number of predictors. The number of predictors for the two modules do not \-match: {num1} != {num2}"""--_ERR_MSG_PREDICTOR_NAMES = """Program predictor names must match at \-corresponding indices for structural equivalency. The predictor names for the \-programs do not match at index {ind}: '{name1}' != '{name2}'"""--_ERR_MSG_SHARED_PREDICTORS = """The programs share the following predictor(s) \-with each other: {pred_names}"""--_ERR_MSG_UNSET_DSPY_LM = """LM consistency violated: Expected all predictors’ \-LMs to be set when dspy.settings.lm is set to ‘NoneType’, but some \-predictors have 'NoneType' LMs.--{lm_info}"""--_ERR_MSG_SET_DSPY_LM = """LM consistency violated: Expected all predictors' \-LMs to be 'NoneType' when 'dspy.settings.lm' is set, but some predictors have \-LMs set.--{lm_info}"""--_ERR_MSG_PROGRAM_LM = """Setting or getting the LM of a program is an \-experimental feature. Please enable the 'dspy.settings.experimental' flag to \-use these feature."""--_INFO_LM_PROGRAM = """The LM set in 'dspy.settings.lm' is an instance of \-'{lname}' LMs set for the predictors in the module are:{pred_lm_info}"""--_INFO_LM_PRED = """- LM for {pname} is an instance of ’{lname}’"""---#--------------------------------------------------------------------------------# Helper functions-#---------------------------------------------------------------------------------def assert_structural_equivalency_for_programs(- program1: object,- program2: object,- ) -> Optional[AssertionError]:- """Assert that the program is structurally equivalent to another program.-- Args:- program1: The program to compare with.- program2: The program to compare with.-- Raises:- AssertionError: If the programs are not structurally equivalent.- """- # TODO: The following import can be removed if "Program" class can be moved- # to its own module, since "Module" is the base class for both the “Program”- # and "Predict" classes.- # Function level import to prevent issues with circular imports.- from dspy.predict.predict import (- assert_structural_equivalency_for_predictors- )-- # Assert that the objects are predictors- assert isinstance(program1, Program)- assert isinstance(program2, Program)-- # Assert that the two programs have the same number of predictors- num1 = len(program1.predictors())- num2 = len(program2.predictors())- err_msg = _ERR_MSG_NUM_PREDICTORS.format(num1=num1, num2=num2)- assert num1 == num2, err_msg-- # Assert that the two programs have structurally equivalent predictors- # sharing the same names- pzip = zip(program1.named_predictors(), program2.named_predictors())- for ind, ((name1, pred1), (name2, pred2)) in enumerate(pzip):- # Check for predictor name match- err_msg = _ERR_MSG_PREDICTOR_NAMES.format(- ind=ind, name1=name1, name2=name2- )- assert name1 == name2, err_msg-- # Check for structural equivalency of the predictors- assert_structural_equivalency_for_predictors(pred1, pred2)---def assert_no_shared_predictor_for_programs(- program1: Program,- program2: Program,- ) -> Optional[AssertionError]:- """Assert that the programs shares no predictor with each other.-- Args:- program1: The program to compare with.- program2: The program to compare with.-- Raises:- AssertionError: If the two programs share predictors.- """- # Get the names and IDs of the predictors in the two programs- id_to_name1 = {id(p): n for n, p in program1.named_predictors()}- id_to_name2 = {id(p): n for n, p in program2.named_predictors()}-- # Find the shared predictors- shared_ids = set(id_to_name1.keys()) & set(id_to_name2.keys())- if shared_ids:- pred_names = ", ".join(id_to_name1[id] for id in shared_ids)- err_msg = _ERR_MSG_SHARED_PREDICTORS.format(pred_names=pred_names)- raise AssertionError(err_msg)---def are_all_predictor_lms_set_for_program(program: Program) -> bool:- """Check if all predictors in the program have their LMs set.-- Returns:- bool: `True` if all predictors have their LMs set, `False` otherwise.- """- return all(pred.lm is not None for pred in program.predictors())---def are_all_predictor_lms_unset_for_program(program: Program) -> bool:- """Check if all predictors in the program have their LMs unset.-- Returns:- bool: True if all predictors have their LMs unset, False otherwise.- """- return all(pred.lm is None for pred in program.predictors())---def get_lm_info_str_for_program(program: Program):- """Get an informational string about the LMs affecting this program."""- # The informational string to be returned- pred_lm_info = ""-- # Get the LM information for each predictor in the program- for pname, pred in program.named_predictors():- lname = pred.lm.__class__.__name__- pred_lm_info += _INFO_LM_PRED.format(pname=pname, lname=lname)-- # Get the LM information from the global setting; augment the information- # string- lname = dspy.settings.lm.__class__.__name__- info = _INFO_LM_PROGRAM.format(lname=lname, pred_lm_info=pred_lm_info)-- return info---def assert_lm_consistency_for_program(- program: Program- ) -> Optional[AssertionError]:- """Assert that the program satisfies the LM consistency property.-- Ensure either that (1) all predictors in the module have their LMs set when- `dspy.settings.lm` is `NoneType`, or, (2) none of the program predictors- have their LMs set when `dspy.settings.lm` is set.-- Raises:- AssertionError: If the LM consistency property is violated.- """- # The error message to be raised if the LM consistency property is violated- err_msg = None-- # Check if the LM consistency property is violated- global_flag = dspy.settings.lm is not None- if not global_flag and not are_all_predictor_lms_set_for_program(program):- err_msg = _ERR_MSG_UNSET_DSPY_LM- elif global_flag and not are_all_predictor_lms_unset_for_program(program):- err_msg = _ERR_MSG_SET_DSPY_LM-- # Raise an error if the LM consistency property is violated- if err_msg is not None:- lm_info = get_lm_info_str_for_program(program)- err_msg = err_msg.format(lm_info=lm_info)- raise AssertionError(err_msg)---#--------------------------------------------------------------------------------# Classes-#-------------------------------------------------------------------------------+from dspy.primitives.utils import _ERR_MSG_PROGRAM_LMclass ProgramMeta(type):pass- # def __call__(cls, *args, **kwargs):- # obj = super(ProgramMeta, cls).__call__(*args, **kwargs)-- # if issubclass(cls, Program) and not getattr(obj, "_program_init_called", False):- # obj._base_init()- # obj._program_init_called = True- # return obj-class Module(BaseModule, metaclass=ProgramMeta):def _base_init(self):@@ -211,7 +21,7 @@ class Module(BaseModule, metaclass=ProgramMeta):def __call__(self, *args, **kwargs):return self.forward(*args, **kwargs)- def set_lm(self, lm: LM) -> Optional[AssertionError]:+ def set_lm(self, lm: LM):"""Set the LM for all predictors in the module.This is an experimental method, which requires dsp.settings.experimentaldiff --git a/dspy/primitives/utils.py b/dspy/primitives/utils.pynew file mode 100644index 000000000..652aab117--- /dev/null+++ b/dspy/primitives/utils.py@@ -0,0 +1,171 @@+from typing import Optional++import dspy+from dspy.primitives.program import Program++_ERR_MSG_NUM_PREDICTORS = """Structurally equivalent programs must have the \+the number of predictors. The number of predictors for the two modules do not \+match: {num1} != {num2}"""++_ERR_MSG_PREDICTOR_NAMES = """Program predictor names must match at \+corresponding indices for structural equivalency. The predictor names for the \+programs do not match at index {ind}: '{name1}' != '{name2}'"""++_ERR_MSG_SHARED_PREDICTORS = """The programs share the following predictor(s) \+with each other: {pred_names}"""++_ERR_MSG_UNSET_DSPY_LM = """LM consistency violated: Expected all predictors’ \+LMs to be set when dspy.settings.lm is set to ‘NoneType’, but some \+predictors have 'NoneType' LMs.++{lm_info}"""++_ERR_MSG_SET_DSPY_LM = """LM consistency violated: Expected all predictors' \+LMs to be 'NoneType' when 'dspy.settings.lm' is set, but some predictors have \+LMs set.++{lm_info}"""++_ERR_MSG_PROGRAM_LM = """Setting or getting the LM of a program is an \+experimental feature. Please enable the 'dspy.settings.experimental' flag to \+use these feature."""++_INFO_LM_PROGRAM = """The LM set in 'dspy.settings.lm' is an instance of \+'{lname}' LMs set for the predictors in the module are:{pred_lm_info}"""++_INFO_LM_PRED = """+ LM for {pname} is an instance of ’{lname}’"""++def assert_structural_equivalency_for_programs(+ program1: object,+ program2: object,+ ) -> Optional[AssertionError]:+ """Assert that the program is structurally equivalent to another program.++ Args:+ program1: The program to compare with.+ program2: The program to compare with.++ Raises:+ AssertionError: If the programs are not structurally equivalent.+ """+ # TODO: The following import can be removed if "Program" class can be moved+ # to its own module, since "Module" is the base class for both the “Program”+ # and "Predict" classes.+ # Function level import to prevent issues with circular imports.+ from dspy.predict.predict import (+ assert_structural_equivalency_for_predictors+ )++ # Assert that the objects are predictors+ assert isinstance(program1, Program)+ assert isinstance(program2, Program)++ # Assert that the two programs have the same number of predictors+ num1 = len(program1.predictors())+ num2 = len(program2.predictors())+ err_msg = _ERR_MSG_NUM_PREDICTORS.format(num1=num1, num2=num2)+ assert num1 == num2, err_msg++ # Assert that the two programs have structurally equivalent predictors+ # sharing the same names+ pzip = zip(program1.named_predictors(), program2.named_predictors())+ for ind, ((name1, pred1), (name2, pred2)) in enumerate(pzip):+ # Check for predictor name match+ err_msg = _ERR_MSG_PREDICTOR_NAMES.format(+ ind=ind, name1=name1, name2=name2+ )+ assert name1 == name2, err_msg++ # Check for structural equivalency of the predictors+ assert_structural_equivalency_for_predictors(pred1, pred2)+++def assert_no_shared_predictor_for_programs(+ program1: Program,+ program2: Program,+ ) -> Optional[AssertionError]:+ """Assert that the programs shares no predictor with each other.++ Args:+ program1: The program to compare with.+ program2: The program to compare with.++ Raises:+ AssertionError: If the two programs share predictors.+ """+ # Get the names and IDs of the predictors in the two programs+ id_to_name1 = {id(p): n for n, p in program1.named_predictors()}+ id_to_name2 = {id(p): n for n, p in program2.named_predictors()}++ # Find the shared predictors+ shared_ids = set(id_to_name1.keys()) & set(id_to_name2.keys())+ if shared_ids:+ pred_names = ", ".join(id_to_name1[id] for id in shared_ids)+ err_msg = _ERR_MSG_SHARED_PREDICTORS.format(pred_names=pred_names)+ raise AssertionError(err_msg)+++def are_all_predictor_lms_set_for_program(program: Program) -> bool:+ """Check if all predictors in the program have their LMs set.++ Returns:+ bool: `True` if all predictors have their LMs set, `False` otherwise.+ """+ return all(pred.lm is not None for pred in program.predictors())+++def are_all_predictor_lms_unset_for_program(program: Program) -> bool:+ """Check if all predictors in the program have their LMs unset.++ Returns:+ bool: True if all predictors have their LMs unset, False otherwise.+ """+ return all(pred.lm is None for pred in program.predictors())+++def get_lm_info_str_for_program(program: Program):+ """Get an informational string about the LMs affecting this program."""+ # The informational string to be returned+ pred_lm_info = ""++ # Get the LM information for each predictor in the program+ for pname, pred in program.named_predictors():+ lname = pred.lm.__class__.__name__+ pred_lm_info += _INFO_LM_PRED.format(pname=pname, lname=lname)++ # Get the LM information from the global setting; augment the information+ # string+ lname = dspy.settings.lm.__class__.__name__+ info = _INFO_LM_PROGRAM.format(lname=lname, pred_lm_info=pred_lm_info)++ return info+++def assert_lm_consistency_for_program(+ program: Program+ ) -> Optional[AssertionError]:+ """Assert that the program satisfies the LM consistency property.++ Ensure either that (1) all predictors in the module have their LMs set when+ `dspy.settings.lm` is `NoneType`, or, (2) none of the program predictors+ have their LMs set when `dspy.settings.lm` is set.++ Raises:+ AssertionError: If the LM consistency property is violated.+ """+ # The error message to be raised if the LM consistency property is violated+ err_msg = None++ # Check if the LM consistency property is violated+ global_flag = dspy.settings.lm is not None+ if not global_flag and not are_all_predictor_lms_set_for_program(program):+ err_msg = _ERR_MSG_UNSET_DSPY_LM+ elif global_flag and not are_all_predictor_lms_unset_for_program(program):+ err_msg = _ERR_MSG_SET_DSPY_LM++ # Raise an error if the LM consistency property is violated+ if err_msg is not None:+ lm_info = get_lm_info_str_for_program(program)+ err_msg = err_msg.format(lm_info=lm_info)+ raise AssertionError(err_msg)\ No newline at end of filediff --git a/dspy/teleprompt/finetune_teleprompter.py b/dspy/teleprompt/finetune_teleprompter.pyindex caecabf4c..d98a32533 100644--- a/dspy/teleprompt/finetune_teleprompter.py+++ b/dspy/teleprompt/finetune_teleprompter.py@@ -156,11 +156,8 @@ def build_messages_from_trace(for pred_ind, (pred, inputs, outputs) in enumerate(trace):# Get the demos from the predictor if exclude_demos is Falsedemos = [] if exclude_demos else pred.demos-messages = adapter.format(pred.signature, demos, inputs)-- formatted_completion = adapter.format_completion(pred.signature, outputs)- messages.append({"role": "assistant", "content": formatted_completion})+ messages.append(adapter.format_turn(signature=pred.signature, values=outputs, role="assistant", incomplete=False))data.append(messages)return data