Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Introduce `dspy.Refine`, `dspy.BestOfN` and fix thread safety for `dspy.ChainOfThoughtWithHint` (#1959)

Chen Qian4f2d2f826079
diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.py
index bc85ba7b1..e57615587 100644
--- a/dspy/predict/__init__.py
+++ b/dspy/predict/__init__.py
@@ -1,15 +1,18 @@
from dspy.predict.aggregation import majority
+from dspy.predict.best_of_n import BestOfN
from dspy.predict.chain_of_thought import ChainOfThought
from dspy.predict.chain_of_thought_with_hint import ChainOfThoughtWithHint
from dspy.predict.knn import KNN
from dspy.predict.multi_chain_comparison import MultiChainComparison
from dspy.predict.predict import Predict
from dspy.predict.program_of_thought import ProgramOfThought
-from dspy.predict.react import ReAct
+from dspy.predict.react import ReAct, Tool
+from dspy.predict.refine import Refine
from dspy.predict.parallel import Parallel
__all__ = [
"majority",
+ "BestOfN",
"ChainOfThought",
"ChainOfThoughtWithHint",
"KNN",
@@ -17,5 +20,7 @@ __all__ = [
"Predict",
"ProgramOfThought",
"ReAct",
+ "Refine",
+ "Tool",
"Parallel",
]
diff --git a/dspy/predict/best_of_n.py b/dspy/predict/best_of_n.py
new file mode 100644
index 000000000..6ca0dd90d
--- /dev/null
+++ b/dspy/predict/best_of_n.py
@@ -0,0 +1,43 @@
+import dspy
+
+from .predict import Module
+from typing import Callable
+
+
+class BestOfN(Module):
+ def __init__(self, module, N: int, reward_fn: Callable, threshold: float):
+ self.module = module
+ self.reward_fn = lambda *args: reward_fn(*args) # to prevent this from becoming a parameter
+ self.threshold = threshold
+ self.N = N
+
+ def forward(self, **kwargs):
+ lm = self.module.get_lm() or dspy.settings.lm
+ temps = [lm.kwargs['temperature']] + [0.5 + i * (0.5 / self.N) for i in range(self.N)]
+ temps = list(dict.fromkeys(temps))[:self.N]
+ best_pred, best_trace, best_reward = None, None, -float("inf")
+
+ for t in temps:
+ lm_ = lm.copy(temperature=t)
+ mod = self.module.deepcopy()
+ mod.set_lm(lm_)
+
+ try:
+ with dspy.context(trace=[]):
+ pred = mod(**kwargs)
+ trace = dspy.settings.trace.copy()
+
+ # NOTE: Not including the trace of reward_fn.
+ reward = self.reward_fn(kwargs, pred)
+
+ if reward > best_reward:
+ best_reward, best_pred, best_trace = reward, pred, trace
+
+ if self.threshold is not None and reward >= self.threshold:
+ break
+
+ except Exception as e:
+ print(f"Attempt failed with temperature {t}: {e}")
+
+ dspy.settings.trace.extend(best_trace)
+ return best_pred
diff --git a/dspy/predict/chain_of_thought_with_hint.py b/dspy/predict/chain_of_thought_with_hint.py
index fa33ca1c6..d7856806f 100644
--- a/dspy/predict/chain_of_thought_with_hint.py
+++ b/dspy/predict/chain_of_thought_with_hint.py
@@ -1,9 +1,6 @@
import dspy
-
from .predict import Module
-# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.
-
class ChainOfThoughtWithHint(Module):
def __init__(self, signature, rationale_type=None, **config):
self.signature = dspy.ensure_signature(signature)
@@ -19,16 +16,11 @@ class ChainOfThoughtWithHint(Module):
kwargs[last_key] = str(kwargs[last_key]) + hint
# Run CoT then update the trace with original kwargs, i.e. without the hint.
- pred = self.module(**kwargs)
- this_trace = dspy.settings.trace[-1]
- dspy.settings.trace[-1] = (this_trace[0], original_kwargs, this_trace[2])
+ with dspy.context(trace=[]):
+ pred = self.module(**kwargs)
+ this_trace = dspy.settings.trace[-1]
+
+ dspy.settings.trace.append((this_trace[0], original_kwargs, this_trace[2]))
return pred
return self.module(**kwargs)
-
-
-"""
-TODO: In principle, we can update the field's prefix during forward too to fill any thing based on the input args.
-
-IF the user didn't overwrite our default rationale_type.
-"""
\ No newline at end of file
diff --git a/dspy/predict/langchain.py b/dspy/predict/langchain.py
deleted file mode 100644
index 223deed28..000000000
--- a/dspy/predict/langchain.py
+++ /dev/null
@@ -1,192 +0,0 @@
-# import copy
-# import random
-# from functools import reduce
-
-# from langchain_core.pydantic_v1 import Extra
-# from langchain_core.runnables import Runnable
-
-# import dspy.dsp as dsp
-# import dspy
-# from dspy.predict.parameter import Parameter
-# from dspy.predict.predict import Predict
-# from dspy.primitives.prediction import Prediction
-# from dspy.signatures.field import OldInputField, OldOutputField
-# from dspy.signatures.signature import infer_prefix
-
-# # TODO: This class is currently hard to test, because it hardcodes gpt-4 usage:
-# # gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat')
-
-# class Template2Signature(dspy.Signature):
-# """You are a processor for prompts. I will give you a prompt template (Python f-string) for an arbitrary task for other LMs.
-# Your job is to prepare three modular pieces: (i) any essential task instructions or guidelines, (ii) a list of variable names for inputs, (iv) the variable name for output."""
-
-# template = dspy.InputField(format=lambda x: f"```\n\n{x.strip()}\n\n```\n\nLet's now prepare three modular pieces.")
-# essential_instructions = dspy.OutputField()
-# input_keys = dspy.OutputField(desc='comma-separated list of valid variable names')
-# output_key = dspy.OutputField(desc='a valid variable name')
-
-
-# class ShallowCopyOnly:
-# def __init__(self, obj): self.obj = obj
-# def __getattr__(self, item): return getattr(self.obj, item)
-# def __deepcopy__(self, memo): return ShallowCopyOnly(copy.copy(self.obj))
-
-
-# class LangChainPredictMetaClass(type(Predict), type(Runnable)):
-# pass
-
-# class LangChainPredict(Predict, Runnable, metaclass=LangChainPredictMetaClass):
-# class Config: extra = Extra.allow # Allow extra attributes that are not defined in the model
-
-# def __init__(self, prompt, llm, **config):
-# Runnable.__init__(self)
-# Parameter.__init__(self)
-
-# self.langchain_llm = ShallowCopyOnly(llm)
-
-# try: langchain_template = '\n'.join([msg.prompt.template for msg in prompt.messages])
-# except AttributeError: langchain_template = prompt.template
-
-# self.stage = random.randbytes(8).hex()
-# self.signature, self.output_field_key = self._build_signature(langchain_template)
-# self.config = config
-# self.reset()
-
-# def reset(self):
-# self.lm = None
-# self.traces = []
-# self.train = []
-# self.demos = []
-
-# def dump_state(self, save_verbose=False):
-# """save_verbose is set as a default argument to support the inherited Parameter interface for dump_state"""
-# state_keys = ["lm", "traces", "train", "demos"]
-# return {k: getattr(self, k) for k in state_keys}
-
-# def load_state(self, state):
-# for name, value in state.items():
-# setattr(self, name, value)
-
-# self.demos = [dspy.Example(**x) for x in self.demos]
-
-# def __call__(self, *arg, **kwargs):
-# if len(arg) > 0: kwargs = {**arg[0], **kwargs}
-# return self.forward(**kwargs)
-
-# def _build_signature(self, template):
-# gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat')
-
-# with dspy.context(lm=gpt4T): parts = dspy.Predict(Template2Signature)(template=template)
-
-# inputs = {k.strip(): OldInputField() for k in parts.input_keys.split(',')}
-# outputs = {k.strip(): OldOutputField() for k in parts.output_key.split(',')}
-
-# for k, v in inputs.items():
-# v.finalize(k, infer_prefix(k)) # TODO: Generate from the template at dspy.Predict(Template2Signature)
-
-# for k, v in outputs.items():
-# output_field_key = k
-# v.finalize(k, infer_prefix(k))
-
-# return dsp.Template(parts.essential_instructions, **inputs, **outputs), output_field_key
-
-# def forward(self, **kwargs):
-# # Extract the three privileged keyword arguments.
-# signature = kwargs.pop("signature", self.signature)
-# demos = kwargs.pop("demos", self.demos)
-# config = dict(**self.config, **kwargs.pop("config", {}))
-
-# prompt = signature(dsp.Example(demos=demos, **kwargs))
-# output = self.langchain_llm.invoke(prompt, **config)
-
-# try: content = output.content
-# except AttributeError: content = output
-
-# pred = Prediction.from_completions([{self.output_field_key: content}], signature=signature)
-
-# # print('#> len(demos) =', len(demos))
-# # print(f"#> {prompt}")
-# # print(f"#> PRED = {content}\n\n\n")
-# dspy.settings.langchain_history.append((prompt, pred))
-
-# if dsp.settings.trace is not None:
-# trace = dsp.settings.trace
-# trace.append((self, {**kwargs}, pred))
-
-# return output
-
-# def invoke(self, d, *args, **kwargs):
-# # print(d)
-# return self.forward(**d)
-
-
-# # Almost good but need output parsing for the fields!
-# # TODO: Use template.extract(example, p)
-
-# # class LangChainOfThought(LangChainPredict):
-# # def __init__(self, signature, **config):
-# # super().__init__(signature, **config)
-
-# # signature = self.signature
-# # *keys, last_key = signature.kwargs.keys()
-# # rationale_type = dsp.Type(prefix="Reasoning: Let's think step by step in order to",
-# # desc="${produce the " + last_key + "}. We ...")
-
-# # extended_kwargs = {key: signature.kwargs[key] for key in keys}
-# # extended_kwargs.update({"rationale": rationale_type, last_key: signature.kwargs[last_key]})
-# # self.extended_signature = dsp.Template(signature.instructions, **extended_kwargs)
-
-# # def forward(self, **kwargs):
-# # signature = self.extended_signature
-# # return super().forward(signature=signature, **kwargs)
-
-
-# class LangChainModule(dspy.Module):
-# def __init__(self, lcel):
-# super().__init__()
-
-# modules = []
-# for name, node in lcel.get_graph().nodes.items():
-# if isinstance(node.data, LangChainPredict): modules.append(node.data)
-
-# self.modules = modules
-# self.chain = lcel
-
-# def forward(self, **kwargs):
-# output_keys = ['output', self.modules[-1].output_field_key]
-# output = self.chain.invoke(dict(**kwargs))
-
-# try: output = output.content
-# except Exception: pass
-
-# return dspy.Prediction({k: output for k in output_keys})
-
-# def invoke(self, d, *args, **kwargs):
-# return self.forward(**d).output
-
-# def reset_copy(self):
-# """Override `reset_copy()` to skip parts that cannot be deep copied."""
-# if self.chain is None:
-# obj = copy.deepcopy(self)
-# else:
-# chain = self.chain
-# steps = []
-# for step in chain.steps:
-# if isinstance(step, LangChainPredict):
-# # Only copy the LangChainPredict object.
-# steps.append(copy.deepcopy(step))
-# else:
-# steps.append(step)
-
-# chain_copy = reduce(lambda x, y: x | y, steps)
-# # Temporarily remove the chain from `self` instance for deep copying.
-# self.chain = None
-# obj = copy.deepcopy(self)
-# obj.chain = chain_copy
-# # Put back the chain to `self` instance.
-# self.chain = chain
-
-# for param in obj.parameters():
-# param.reset()
-
-# return obj
diff --git a/dspy/predict/llamaindex.py b/dspy/predict/llamaindex.py
deleted file mode 100644
index b261cf6c8..000000000
--- a/dspy/predict/llamaindex.py
+++ /dev/null
@@ -1,263 +0,0 @@
-# import re
-# from copy import deepcopy
-# from typing import Any, Callable, Dict, List, Optional
-
-# from llama_index.core.base.llms.base import BaseLLM
-# from llama_index.core.base.llms.generic_utils import (
-# prompt_to_messages,
-# )
-# from llama_index.core.base.llms.types import ChatMessage
-# from llama_index.core.base.query_pipeline.query import InputKeys, OutputKeys, QueryComponent
-# from llama_index.core.callbacks.base import CallbackManager
-# from llama_index.core.prompts import BasePromptTemplate, PromptTemplate
-# from llama_index.core.query_pipeline import QueryPipeline
-
-# import dspy.dsp as dsp
-# import dspy
-# from dspy import Predict
-# from dspy.signatures.field import InputField, OutputField
-# from dspy.signatures.signature import ensure_signature, make_signature, signature_to_template
-
-
-# def get_formatted_template(predict_module: Predict, kwargs: Dict[str, Any]) -> str:
-# """Get formatted template from predict module."""
-# # Extract the three privileged keyword arguments.
-# signature = ensure_signature(predict_module.signature)
-# demos = predict_module.demos
-
-# # All of the other kwargs are presumed to fit a prefix of the signature.
-# # That is, they are input variables for the bottom most generation, so
-# # we place them inside the input - x - together with the demos.
-# x = dsp.Example(demos=demos, **kwargs)
-
-# # Switch to legacy format for dsp.generate
-# template = signature_to_template(signature)
-
-# return template(x)
-
-
-# def replace_placeholder(text: str) -> str:
-# # Use a regular expression to find and replace ${...} with ${{...}}
-# return re.sub(r'\$\{([^\{\}]*)\}', r'${{\1}}', text)
-
-
-# def _input_keys_from_template(template: dsp.Template) -> InputKeys:
-# """Get input keys from template."""
-# # get only fields that are marked OldInputField and NOT OldOutputField
-# # template_vars = list(template.kwargs.keys())
-# return [
-# k for k, v in template.kwargs.items() if isinstance(v, dspy.signatures.OldInputField)
-# ]
-
-# def _output_keys_from_template(template: dsp.Template) -> InputKeys:
-# """Get output keys from template."""
-# # get only fields that are marked OldOutputField and NOT OldInputField
-# # template_vars = list(template.kwargs.keys())
-# return [
-# k for k, v in template.kwargs.items() if isinstance(v, dspy.signatures.OldOutputField)
-# ]
-
-
-# class DSPyPromptTemplate(BasePromptTemplate):
-# """A prompt template for DSPy.
-
-# Takes in a predict module from DSPy (whether unoptimized or optimized),
-# and extracts the relevant prompt template from it given the input.
-
-# """
-
-# predict_module: Predict
-
-# def __init__(
-# self,
-# predict_module: Predict,
-# metadata: Optional[Dict[str, Any]] = None,
-# template_var_mappings: Optional[Dict[str, Any]] = None,
-# function_mappings: Optional[Dict[str, Callable]] = None,
-# **kwargs: Any,
-# ) -> None:
-# template = signature_to_template(predict_module.signature)
-# template_vars = _input_keys_from_template(template)
-# # print(f"TEMPLATE VARS: {template_vars}")
-# # raise Exception
-
-# super().__init__(
-# predict_module=predict_module,
-# metadata=metadata or {},
-# template_vars=template_vars,
-# kwargs=kwargs,
-# template_var_mappings=template_var_mappings,
-# function_mappings=function_mappings,
-# )
-
-# def partial_format(self, **kwargs: Any) -> "BasePromptTemplate":
-# """Returns a new prompt template with the provided kwargs."""
-# # NOTE: this is a copy of the implementation in `PromptTemplate`
-# output_parser = self.output_parser
-# self.output_parser = None
-
-# # get function and fixed kwargs, and add that to a copy
-# # of the current prompt object
-# prompt = deepcopy(self)
-# prompt.kwargs.update(kwargs)
-
-# # NOTE: put the output parser back
-# prompt.output_parser = output_parser
-# self.output_parser = output_parser
-# return prompt
-
-# def format(self, llm: Optional[BaseLLM] = None, **kwargs: Any) -> str:
-# """Formats the prompt template."""
-# mapped_kwargs = self._map_all_vars(kwargs)
-# return get_formatted_template(self.predict_module, mapped_kwargs)
-
-# def format_messages(
-# self, llm: Optional[BaseLLM] = None, **kwargs: Any,
-# ) -> List[ChatMessage]:
-# """Formats the prompt template into chat messages."""
-# del llm # unused
-# prompt = self.format(**kwargs)
-# return prompt_to_messages(prompt)
-
-# def get_template(self, llm: Optional[BaseLLM] = None) -> str:
-# """Get template."""
-# # get kwarg templates
-# kwarg_tmpl_map = {k: "{k}" for k in self.template_vars}
-
-# # get "raw" template with all the values filled in with {var_name}
-# template0 = get_formatted_template(self.predict_module, kwarg_tmpl_map)
-# # HACK: there are special 'format' variables of the form ${var_name} that are meant to
-# # prompt the LLM, but we do NOT want to replace with actual prompt variable values.
-# # Replace those with double brackets
-# template1 = replace_placeholder(template0)
-
-# return template1
-
-
-# # copied from langchain.py
-# class Template2Signature(dspy.Signature):
-# """You are a processor for prompts. I will give you a prompt template (Python f-string) for an arbitrary task for other LMs.
-# Your job is to prepare three modular pieces: (i) any essential task instructions or guidelines, (ii) a list of variable names for inputs, (iv) the variable name for output."""
-
-# template = dspy.InputField(format=lambda x: f"```\n\n{x.strip()}\n\n```\n\nLet's now prepare three modular pieces.")
-# essential_instructions = dspy.OutputField()
-# input_keys = dspy.OutputField(desc='comma-separated list of valid variable names')
-# output_key = dspy.OutputField(desc='a valid variable name')
-
-
-# def build_signature(prompt: PromptTemplate) -> dspy.Signature:
-# """Attempt to build signature from prompt."""
-# # TODO: allow plugging in any llamaindex LLM
-# gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat')
-
-# with dspy.context(lm=gpt4T):
-# parts = dspy.Predict(Template2Signature)(template=prompt.template)
-
-# inputs = {k.strip(): InputField() for k in parts.input_keys.split(',')}
-# outputs = {k.strip(): OutputField() for k in parts.output_key.split(',')}
-
-# # dynamically create a pydantic model that subclasses dspy.Signature
-# fields = {
-# k: (str, v) for k, v in {**inputs, **outputs}.items()
-# }
-# signature = make_signature(fields, parts.essential_instructions)
-# return signature
-
-
-# class DSPyComponent(QueryComponent):
-# """DSPy Query Component.
-
-# Can take in either a predict module directly.
-# TODO: add ability to translate from an existing prompt template / LLM.
-
-# """
-# predict_module: dspy.Predict
-# predict_template: dsp.Template
-
-# class Config:
-# arbitrary_types_allowed = True
-
-# def __init__(
-# self,
-# predict_module: dspy.Predict,
-# ) -> None:
-# """Initialize."""
-# return super().__init__(
-# predict_module=predict_module,
-# predict_template=signature_to_template(predict_module.signature),
-# )
-
-# @classmethod
-# def from_prompt(
-# cls,
-# prompt_template: BasePromptTemplate,
-# # llm: BaseLLM,
-# ) -> "DSPyComponent":
-# """Initialize from prompt template.
-
-# LLM is a TODO - currently use DSPy LLM classes.
-
-# """
-# signature = build_signature(prompt_template)
-# predict_module = Predict(signature)
-# return cls(predict_module=predict_module)
-
-# def set_callback_manager(self, callback_manager: CallbackManager) -> None:
-# """Set callback manager."""
-# # TODO: implement
-# pass
-
-# def _validate_component_inputs(self, input: Dict[str, Any]) -> Dict[str, Any]:
-# """Validate component inputs during run_component."""
-# return input
-
-# def _run_component(self, **kwargs: Any) -> Dict:
-# """Run component."""
-# prediction = self.predict_module(**kwargs)
-# return {
-# k: getattr(prediction, k) for k in self.output_keys.required_keys
-# }
-
-# async def _arun_component(self, **kwargs: Any) -> Any:
-# """Run component (async)."""
-# # TODO: no async predict module yet
-# return self._run_component(**kwargs)
-
-# @property
-# def input_keys(self) -> InputKeys:
-# """Input keys."""
-# input_keys = _input_keys_from_template(self.predict_template)
-# return InputKeys.from_keys(input_keys)
-
-# @property
-# def output_keys(self) -> OutputKeys:
-# """Output keys."""
-# output_keys = _output_keys_from_template(self.predict_template)
-# return OutputKeys.from_keys(output_keys)
-
-
-# class LlamaIndexModule(dspy.Module):
-# """A module for LlamaIndex.
-
-# Wraps a QueryPipeline and exposes it as a dspy module for optimization.
-
-# """
-
-# class Config:
-# arbitrary_types_allowed = True
-
-# def __init__(self, query_pipeline: QueryPipeline) -> None:
-# """Initialize."""
-# super().__init__()
-# self.query_pipeline = query_pipeline
-# self.predict_modules = []
-# for module in query_pipeline.module_dict.values():
-# if isinstance(module, DSPyComponent):
-# self.predict_modules.append(module.predict_module)
-
-
-# def forward(self, **kwargs: Any) -> Dict[str, Any]:
-# """Forward."""
-# output_dict = self.query_pipeline.run(**kwargs, return_values_direct=False)
-# return dspy.Prediction(**output_dict)
-
diff --git a/dspy/predict/refine.py b/dspy/predict/refine.py
new file mode 100644
index 000000000..8d54f8d13
--- /dev/null
+++ b/dspy/predict/refine.py
@@ -0,0 +1,162 @@
+import dspy
+import ujson
+import inspect
+import textwrap
+
+from .predict import Module
+from dspy.signatures import Signature, InputField, OutputField
+
+from typing import Callable
+from dspy.adapters.chat_adapter import enumerate_fields
+
+
+class OfferFeedback(Signature):
+ """
+ In the discussion, assign blame to each module that contributed to the final reward being below the threshold, if
+ any. Then, prescribe concrete advice of how the module should act on its future input when we retry the process, if
+ it were to receive the same or similar inputs. If a module is not to blame, the advice should be N/A.
+ The module will not see its own history, so it needs to rely on entirely concrete and actionable advice from you
+ to avoid the same mistake on the same or similar inputs.
+ """
+
+ program_code: str = InputField(desc="The code of the program that we are analyzing")
+ modules_defn: str = InputField(desc="The definition of each module in the program, including its I/O")
+ program_inputs: str = InputField(desc="The inputs to the program that we are analyzing")
+ program_trajectory: str = InputField(
+ desc="The trajectory of the program's execution, showing each module's I/O"
+ )
+ program_outputs: str = InputField(desc="The outputs of the program that we are analyzing")
+ reward_code: str = InputField(desc="The code of the reward function that we are analyzing")
+ target_threshold: float = InputField(desc="The target threshold for the reward function")
+ reward_value: float = InputField(desc="The reward value assigned to the program's outputs")
+ module_names: list[str] = InputField(desc="The names of the modules in the program, for which we seek advice")
+ discussion: str = OutputField(desc="Discussing blame of where each module went wrong, if it did")
+ advice: dict[str, str] = OutputField(
+ desc="For each module, describe very concretely, in this order: the specific scenarios in which it has made "
+ "mistakes in the past and what each mistake was, followed by what it should do differently in that kind of"
+ "scenario in the future. If the module is not to blame, write N/A."
+ )
+
+
+class Refine(Module):
+ def __init__(self, module, N: int, reward_fn: Callable, threshold: float):
+ self.module = module
+ self.reward_fn = lambda *args: reward_fn(*args) # to prevent this from becoming a parameter
+ self.threshold = threshold
+ self.N = N
+
+ self.module_code = inspect.getsource(module.__class__)
+ try:
+ self.reward_fn_code = inspect.getsource(reward_fn)
+ except TypeError:
+ self.reward_fn_code = inspect.getsource(reward_fn.__class__)
+
+ def forward(self, **kwargs):
+ lm = self.module.get_lm() or dspy.settings.lm
+ temps = [lm.kwargs["temperature"]] + [0.5 + i * (0.5 / self.N) for i in range(self.N)]
+ temps = list(dict.fromkeys(temps))[: self.N]
+ best_pred, best_trace, best_reward = None, None, -float("inf")
+ advice = None
+ adapter = dspy.settings.adapter or dspy.ChatAdapter()
+
+ for idx, t in enumerate(temps):
+ lm_ = lm.copy(temperature=t)
+ mod = self.module.deepcopy()
+ mod.set_lm(lm_)
+
+ predictor2name = {predictor: name for name, predictor in mod.named_predictors()}
+ signature2name = {predictor.signature: name for name, predictor in mod.named_predictors()}
+ module_names = [name for name, _ in mod.named_predictors()]
+
+ try:
+ with dspy.context(trace=[]):
+ if not advice:
+ outputs = mod(**kwargs)
+ else:
+
+ class WrapperAdapter(adapter.__class__):
+ def __call__(self, lm, lm_kwargs, signature, demos, inputs):
+ inputs["hint_"] = advice.get(signature2name[signature], "N/A")
+ signature = signature.append(
+ "hint_", InputField(desc="A hint to the module from an earlier run")
+ )
+ return adapter(lm, lm_kwargs, signature, demos, inputs)
+
+ with dspy.context(adapter=WrapperAdapter()):
+ outputs = mod(**kwargs)
+
+ trace = dspy.settings.trace.copy()
+
+ # TODO: Remove the hint from the trace, if it's there.
+
+ # NOTE: Not including the trace of reward_fn.
+ reward = self.reward_fn(kwargs, outputs)
+
+ if reward > best_reward:
+ best_reward, best_pred, best_trace = reward, outputs, trace
+
+ if self.threshold is not None and reward >= self.threshold:
+ break
+
+ if idx == self.N - 1:
+ break
+
+ modules = dict(program_code=self.module_code, modules_defn=inspect_modules(mod))
+ trajectory = [dict(module_name=predictor2name[p], inputs=i, outputs=dict(o)) for p, i, o in trace]
+ trajectory = dict(program_inputs=kwargs, program_trajectory=trajectory, program_outputs=dict(outputs))
+ reward = dict(reward_code=self.reward_fn_code, target_threshold=self.threshold, reward_value=reward)
+
+ advise_kwargs = dict(**modules, **trajectory, **reward, module_names=module_names)
+ # advise_kwargs = {k: ujson.dumps(recursive_mask(v), indent=2) for k, v in advise_kwargs.items()}
+ # only dumps if it's a list or dict
+ advise_kwargs = {k: v if isinstance(v, str) else ujson.dumps(recursive_mask(v), indent=2) for k, v in advise_kwargs.items()}
+ advice = dspy.Predict(OfferFeedback)(**advise_kwargs).advice
+ # print(f"Advice for each module: {advice}")
+
+ except Exception as e:
+ print(f"Attempt failed with temperature {t}: {e}")
+
+ dspy.settings.trace.extend(best_trace)
+ return best_pred
+
+
+def inspect_modules(program):
+ separator = "-" * 80
+ output = [separator]
+
+ for idx, (name, predictor) in enumerate(program.named_predictors()):
+ signature = predictor.signature
+ instructions = textwrap.dedent(signature.instructions)
+ instructions = ("\n" + "\t" * 2).join([""] + instructions.splitlines())
+
+ output.append(f"Module {name}")
+ output.append("\n\tInput Fields:")
+ output.append(("\n" + "\t" * 2).join([""] + enumerate_fields(signature.input_fields).splitlines()))
+ output.append("\tOutput Fields:")
+ output.append(("\n" + "\t" * 2).join([""] + enumerate_fields(signature.output_fields).splitlines()))
+ output.append(f"\tOriginal Instructions: {instructions}")
+ output.append(separator)
+
+ return "\n".join([o.strip("\n") for o in output])
+
+
+def recursive_mask(o):
+ # If the object is already serializable, return it.
+ try:
+ ujson.dumps(o)
+ return o
+ except TypeError:
+ pass
+
+ # If it's a dictionary, apply recursively to its values.
+ if isinstance(o, dict):
+ return {k: recursive_mask(v) for k, v in o.items()}
+ # If it's a list, apply recursively.
+ elif isinstance(o, list):
+ return [recursive_mask(v) for v in o]
+ # If it's a tuple, apply recursively.
+ elif isinstance(o, tuple):
+ return tuple(recursive_mask(v) for v in o)
+ # Otherwise, replace it with a placeholder string (or use repr(o)).
+ else:
+ return f"<non-serializable: {type(o).__name__}>"