DSPy: The framework for programming—not prompting—language models
Python
new Signature class + tests + types
Thomas Dybdahl Ahle2d218aa0588a
diff --git a/.gitignore b/.gitignoreindex c47837feb..7d9919d9a 100644--- a/.gitignore+++ b/.gitignore@@ -13,6 +13,9 @@ __pycache__/*.py[cod]*$py.class+# Vim+*.swp+# Jupyter Notebook.ipynb_checkpoints# notebooks/@@ -42,4 +45,4 @@ finetuning_ckpts/.ideaassertion.log*.log-*.db\ No newline at end of file+*.dbdiff --git a/dsp/modules/aws_lm.py b/dsp/modules/aws_lm.pyindex 00906282a..05a733851 100644--- a/dsp/modules/aws_lm.py+++ b/dsp/modules/aws_lm.py@@ -28,7 +28,6 @@ class AWSLM(LM):max_new_tokens: int,truncate_long_prompts: bool = False,input_output_ratio: int = 3,- batch_n: bool = True,) -> None:"""_summary_@@ -41,7 +40,6 @@ class AWSLM(LM):input_output_ratio (int, optional): The rough size of the number of input tokens to output tokens in the worst case. Defaults to 3.temperature (float, optional): _description_. Defaults to 0.0.truncate_long_prompts (bool, optional): If True, remove extremely long inputs to context. Defaults to False.- batch_n (bool, False): If False, call the LM N times rather than batching. Not all AWS models support the n parameter."""super().__init__(model=model)# AWS doesn't have an equivalent of max_tokens so let's clarify@@ -50,10 +48,9 @@ class AWSLM(LM):self._max_new_tokens: int = max_new_tokensself._model_name: str = modelself._truncate_long_prompt_prompts: bool = truncate_long_prompts- self._batch_n: bool = batch_nimport boto3-+self.predictor = boto3.client(service_name, region_name=region_name)@abstractmethod@@ -75,7 +72,7 @@ class AWSLM(LM):return query_kwargs@abstractmethod- def _call_model(self, body: str) -> str | list[str]:+ def _call_model(self, body: str) -> str:"""Call model, get generated input without the formatted prompt"""pass@@ -85,20 +82,7 @@ class AWSLM(LM):) -> dict[str, str | float | int]:pass- def _simple_api_call(self, formatted_prompt: str, **kwargs) -> str | list[str]:- body = self._create_body(formatted_prompt, **kwargs)- json_body = json.dumps(body)- llm_out: str | list[str] = self._call_model(json_body)- if isinstance(llm_out, str):- llm_out = llm_out.replace(formatted_prompt, "")- else:- llm_out = [generated.replace(formatted_prompt, "") for generated in llm_out]- self.history.append(- {"prompt": formatted_prompt, "response": llm_out, "kwargs": body}- )- return llm_out-- def basic_request(self, prompt, **kwargs) -> str | list[str]:+ def basic_request(self, prompt, **kwargs) -> str:"""Query the endpoint."""# Remove any texts that are too long@@ -108,28 +92,16 @@ class AWSLM(LM):formatted_prompt = self._format_prompt(truncated_prompt)else:formatted_prompt = self._format_prompt((prompt))+ body = self._create_body(formatted_prompt, **kwargs)+ json_body: str = json.dumps(body)- llm_out: str | list[str]- if "n" in kwargs.keys():- if self._batch_n:- llm_out = self._simple_api_call(- formatted_prompt=formatted_prompt, **kwargs- )- else:- del kwargs["n"]- llm_out = []- for _ in range(0, kwargs["n"]):- generated: str | list[str] = self._simple_api_call(- formatted_prompt=formatted_prompt, **kwargs- )- if isinstance(generated, str):- llm_out.append(generated)- else:- raise TypeError("Error, list type was returned from LM call")- else:- llm_out = self._simple_api_call(formatted_prompt=formatted_prompt, **kwargs)+ generated: str = self._call_model(json_body)++ self.history.append(+ {"prompt": formatted_prompt, "response": generated, "kwargs": body}+ )- return llm_out+ return generated.replace(formatted_prompt, "")def _estimate_tokens(self, text: str) -> int:return len(text) * CHARS2TOKENSdiff --git a/dsp/templates/template_v2.py b/dsp/templates/template_v2.pyindex 0e1b36885..f61085dd0 100644--- a/dsp/templates/template_v2.py+++ b/dsp/templates/template_v2.py@@ -91,8 +91,8 @@ class TemplateV2:if field.input_variable in self.format_handlers:format_handler = self.format_handlers[field.input_variable]else:-def format_handler(x):+ assert type(x) == str, f"Need format_handler for {field.input_variable} of type {type(x)}"return " ".join(x.split())formatted_value = format_handler(example[field.input_variable])diff --git a/dspy/functional/__init__.py b/dspy/functional/__init__.pynew file mode 100644index 000000000..11fb1bc4d--- /dev/null+++ b/dspy/functional/__init__.py@@ -0,0 +1 @@+from .functional import cot, predictor, FunctionalModule, TypedPredictordiff --git a/dspy/functional/functional.py b/dspy/functional/functional.pynew file mode 100644index 000000000..c4388ee86--- /dev/null+++ b/dspy/functional/functional.py@@ -0,0 +1,327 @@+import inspect, os, openai, dspy, typing, pydantic+from typing import Annotated+import typing+from dsp.templates import passages2text+import json+++MAX_RETRIES = 3+++def predictor(func):+ signature = _func_to_signature(func)+ return TypedPredictor(signature, chain_of_thought=False, simple_output=True)+++def cot(func):+ signature = _func_to_signature(func)+ return TypedPredictor(signature, chain_of_thought=True, simple_output=True)+++class FunctionalModule(dspy.Module):+ def __init__(self):+ super().__init__()+ for name in dir(self):+ attr = getattr(self, name)+ if isinstance(attr, dspy.Module):+ self.__dict__[name] = attr.copy()+++class TypedPredictor(dspy.Module):+ def __init__(self, signature, chain_of_thought=False, simple_output=False):+ super().__init__()+ self.signature = signature+ self.predictor = dspy.Predict(signature)+ self.chain_of_thought = chain_of_thought+ self.simple_output = simple_output++ def copy(self):+ return TypedPredictor(self.signature, self.chain_of_thought, self.simple_output)++ def _prepare_signature(self):+ """Add formats and parsers to the signature fields, based on the type+ annotations of the fields."""+ signature = self.signature+ for name, field in self.signature.fields.items():+ is_output = field.json_schema_extra["__dspy_field_type"] == "output"+ type_ = field.annotation+ if is_output:+ if type_ in (str, int, float, bool):+ signature = signature.with_updated_fields(+ name,+ desc=field.json_schema_extra.get("desc", "")+ + (f". Respond with a single {type_.__name__} value"),+ format=lambda x: x if isinstance(x, str) else str(x),+ parser=type_,+ )+ else:+ # Anything else we wrap in a pydantic object+ unwrap = lambda x: x+ if not inspect.isclass(type_) or not issubclass(+ type_, pydantic.BaseModel+ ):+ type_ = pydantic.create_model(+ "Output", value=(type_, ...), __base__=pydantic.BaseModel+ )+ unwrap = lambda x: x.value+ signature = signature.with_updated_fields(+ name,+ desc=field.json_schema_extra.get("desc", "")+ + (+ f". Respond with a single JSON object using the schema "+ + json.dumps(type_.model_json_schema())+ ),+ format=lambda x: x if isinstance(x, str) else x.json(),+ parser=lambda x: unwrap(+ type_.model_validate_json(_unwrap_json(x))+ ),+ )+ else: # If input field+ format = lambda x: x if isinstance(x, str) else str(x)+ if type_ in (list[str], tuple[str]):+ format = passages2text+ elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):+ format = lambda x: x if isinstance(x, str) else x.json()+ signature = signature.with_updated_fields(name, format=format)++ if self.chain_of_thought:+ output_keys = ", ".join(signature.output_fields.keys())+ signature = signature.prepend(+ "reasoning",+ dspy.OutputField(+ prefix="Reasoning: Let's think step by step in order to",+ desc="${produce the " + output_keys + "}. We ...",+ ),+ )+ return signature++ def forward(self, **kwargs):+ modified_kwargs = kwargs.copy()+ signature = self._prepare_signature()+ for try_i in range(MAX_RETRIES):+ result = self.predictor(**modified_kwargs, new_signature=signature)+ errors = {}+ parsed_results = {}+ # Parse the outputs+ for name, field in signature.output_fields.items():+ try:+ value = getattr(result, name)+ parser = field.json_schema_extra.get("parser", lambda x: x)+ parsed_results[name] = parser(value)+ except (pydantic.ValidationError, ValueError) as e:+ errors[name] = e+ if errors:+ # Add new fields for each error+ for name, error in errors.items():+ modified_kwargs[f"error_{name}_{try_i}"] = str(error)+ signature = signature.append(+ f"error_{name}_{try_i}",+ dspy.InputField(+ prefix=f"Past Error "+ + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),+ desc="An error to avoid in the future",+ ),+ )+ else:+ # If there are no errors, we return the parsed results+ for name, value in parsed_results.items():+ setattr(result, name, value)+ if self.simple_output:+ *_, last_output = signature.output_fields.keys()+ return result[last_output]+ return result+ raise ValueError("Too many retries")+++def _func_to_signature(func):+ """Make a dspy.Signature based on a function definition."""+ sig = inspect.signature(func)+ annotations = typing.get_type_hints(func)+ output_key = func.__name__+ instructions = func.__doc__+ fields = {}++ # Input fields+ for param in sig.parameters.values():+ if param.name == "self":+ continue+ # We default to str as the type of the input+ annotation = annotations.get(param.name, str)+ kwargs = {}+ if typing.get_origin(annotation) is Annotated:+ annotation, kwargs["desc"] = typing.get_args(annotation)+ fields[param.name] = (annotation, dspy.InputField(**kwargs))++ # Output field+ kwargs = {}+ annotation = annotations.get("return", str)+ if typing.get_origin(annotation) is Annotated:+ annotation, kwargs["desc"] = typing.get_args(annotation)+ fields[output_key] = (annotation, dspy.OutputField(**kwargs))++ return dspy.Signature(fields, instructions)+++def _unwrap_json(output):+ output = output.strip()+ if output.startswith("```"):+ if not output.startswith("```json"):+ raise ValueError("json output should start with ```json")+ if not output.endswith("```"):+ raise ValueError("json output should end with ```")+ output = output[7:-3].strip()+ if not output.startswith("{") or not output.endswith("}"):+ raise ValueError("json output should start and end with { and }")+ return output+++################################################################################+# Example usage+################################################################################+++def main():+ class Answer(pydantic.BaseModel):+ value: float+ certainty: float+ comments: list[str] = pydantic.Field(+ description="At least two comments about the answer"+ )++ class QA(dspy.Module):+ @predictor+ def hard_question(self, topic: str) -> str:+ """Think of a hard factual question about a topic. It should be answerable with a number."""++ @cot+ def answer(self, question: Annotated[str, "Question to answer"]) -> Answer:+ pass++ def forward(self, **kwargs):+ question = self.hard_question(**kwargs)+ return (question, self.answer(question=question))++ openai.api_key = os.getenv("OPENAI_API_KEY")+ lm = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=4000)+ # lm = dspy.OpenAI(model="gpt-4", max_tokens=4000)+ # lm = dspy.OpenAI(model="gpt-4-preview-1106", max_tokens=4000)+ with dspy.context(lm=lm):+ qa = QA()+ question, answer = qa(topic="Physics")+ # lm.inspect_history(n=5)++ print("Question:", question)+ print("Answer:", answer)+++################################################################################+# HotpotQA example with SimpleBaleen+################################################################################+++def validate_context_and_answer_and_hops(example, pred, trace=None):+ if not dspy.evaluate.answer_exact_match(example, pred):+ return False+ if not dspy.evaluate.answer_passage_match(example, pred):+ return False++ hops = [example.question] + [+ outputs.query for *_, outputs in trace if "query" in outputs+ ]++ if max([len(h) for h in hops]) > 100:+ return False+ if any(+ dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)+ for idx in range(2, len(hops))+ ):+ return False++ return True+++def gold_passages_retrieved(example, pred, trace=None):+ gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))+ found_titles = set(+ map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context])+ )++ return gold_titles.issubset(found_titles)+++def hotpot():+ from dsp.utils import deduplicate+ import dspy.evaluate+ from dspy.datasets import HotPotQA+ from dspy.evaluate.evaluate import Evaluate+ from dspy.teleprompt.bootstrap import BootstrapFewShot++ print("Load the dataset.")+ dataset = HotPotQA(+ train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0+ )+ trainset = [x.with_inputs("question") for x in dataset.train]+ devset = [x.with_inputs("question") for x in dataset.dev]+ print("Done")++ class SimplifiedBaleen(FunctionalModule):+ def __init__(self, passages_per_hop=3, max_hops=1):+ super().__init__()+ self.retrieve = dspy.Retrieve(k=passages_per_hop)+ self.max_hops = max_hops++ @cot+ def generate_query(self, context: list[str], question) -> str:+ """Write a simple search query that will help answer a complex question."""+ pass++ @cot+ def generate_answer(self, context: list[str], question) -> str:+ """Answer questions with short factoid answers."""+ pass++ def forward(self, question):+ context = []++ for hop in range(self.max_hops):+ query = self.generate_query(context=context, question=question)+ passages = self.retrieve(query).passages+ context = deduplicate(context + passages)++ answer = self.generate_answer(context=context, question=question)+ return dspy.Prediction(context=context, answer=answer)++ openai.api_key = os.getenv("OPENAI_API_KEY")+ rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")+ lm = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=4000)+ dspy.settings.configure(lm=lm, rm=rm, trace=[])++ evaluate_on_hotpotqa = Evaluate(+ devset=devset, num_threads=10, display_progress=True, display_table=5+ )++ # uncompiled (i.e., zero-shot) program+ uncompiled_baleen = SimplifiedBaleen()+ print(+ "Uncompiled Baleen retrieval score:",+ evaluate_on_hotpotqa(uncompiled_baleen, metric=gold_passages_retrieved),+ )++ # compiled (i.e., few-shot) program+ compiled_baleen = BootstrapFewShot(+ metric=validate_context_and_answer_and_hops+ ).compile(+ SimplifiedBaleen(),+ teacher=SimplifiedBaleen(passages_per_hop=2),+ trainset=trainset,+ )+ print(+ "Compiled Baleen retrieval score:",+ evaluate_on_hotpotqa(compiled_baleen, metric=gold_passages_retrieved),+ )+ # lm.inspect_history(n=5)+++if __name__ == "__main__":+ # main()+ hotpot()diff --git a/dspy/predict/__init__.py b/dspy/predict/__init__.pyindex f646ab69f..8b0770150 100644--- a/dspy/predict/__init__.py+++ b/dspy/predict/__init__.py@@ -6,3 +6,4 @@ from .react import ReActfrom .aggregation import majorityfrom .program_of_thought import ProgramOfThoughtfrom .retry import Retry+from .knn import KNN\ No newline at end of filediff --git a/dspy/predict/aggregation.py b/dspy/predict/aggregation.pyindex 2212900c2..ca4154aa2 100644--- a/dspy/predict/aggregation.py+++ b/dspy/predict/aggregation.py@@ -26,10 +26,11 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)except:signature = None- try:- field = field if field else signature.fields[-1].output_variable- except:- field = field if field else list(completions[0].keys())[-1]+ if not field:+ if signature:+ field = signature.output_fields[-1]+ else:+ field = list(completions[0].keys())[-1]# Normalizenormalize = normalize if normalize else lambda x: x@@ -51,5 +52,4 @@ def majority(prediction_or_completions, normalize=default_normalize, field=None)# if input_type == Prediction:return Prediction.from_completions([completion], signature=signature)- return Completions([completion], signature=signature)diff --git a/dspy/predict/chain_of_thought.py b/dspy/predict/chain_of_thought.pyindex fceb3b651..7d50d8a56 100644--- a/dspy/predict/chain_of_thought.py+++ b/dspy/predict/chain_of_thought.py@@ -1,6 +1,7 @@-import dsp+import dsp, dspy+from dspy.signatures.signature import ensure_signature-from .predict import Predict+from .predict import Predict, signature_to_template# TODO: FIXME: Insert this right before the *first* output field. Also rewrite this to use the new signature system.@@ -33,24 +34,15 @@ class ChainOfThought(Predict):self.activated = activated- signature = self.signature- *keys, last_key = signature.kwargs.keys()+ signature = ensure_signature(self.signature)+ *_keys, last_key = signature.output_fields.keys()- DEFAULT_RATIONALE_TYPE = dsp.Type(+ rationale_type = rationale_type or dspy.OutputField(prefix="Reasoning: Let's think step by step in order to",desc="${produce the " + last_key + "}. We ...",)- rationale_type = rationale_type or DEFAULT_RATIONALE_TYPE-- extended_kwargs = {key: signature.kwargs[key] for key in keys}- extended_kwargs.update(- {"rationale": rationale_type, last_key: signature.kwargs[last_key]}- )-- self.extended_signature = dsp.Template(- signature.instructions, **extended_kwargs- )+ self.extended_signature = signature.prepend("rationale", rationale_type, type_=str)def forward(self, **kwargs):new_signature = kwargs.pop("new_signature", None)@@ -62,7 +54,8 @@ class ChainOfThought(Predict):else:signature = self.signatureelse:- signature = dsp.Template(self.signature.instructions, **new_signature)+ signature = new_signature+ # template = dsp.Template(self.signature.instructions, **new_signature)return super().forward(signature=signature, **kwargs)diff --git a/dspy/predict/chain_of_thought_with_hint.py b/dspy/predict/chain_of_thought_with_hint.pyindex b968d0bd9..83d5b5b4b 100644--- a/dspy/predict/chain_of_thought_with_hint.py+++ b/dspy/predict/chain_of_thought_with_hint.py@@ -1,4 +1,4 @@-import dsp+import dsp, dspyfrom .predict import Predict@@ -9,27 +9,18 @@ from .predict import Predictclass ChainOfThoughtWithHint(Predict):def __init__(self, signature, rationale_type=None, activated=True, **config):super().__init__(signature, **config)-self.activated = activated-signature = self.signature- *keys, last_key = signature.kwargs.keys()-- DEFAULT_HINT_TYPE = dsp.Type(prefix="Hint:", desc="${hint}")- DEFAULT_RATIONALE_TYPE = dsp.Type(prefix="Reasoning: Let's think step by step in order to",- desc="${produce the " + last_key + "}. We ...")+ *keys, last_key = signature.fields.keys()+ rationale_type = rationale_type or dspy.OutputField(+ prefix="Reasoning: Let's think step by step in order to",+ desc="${produce the " + last_key + "}. We ...",+ )+ self.extended_signature1 = self.signature.insert(-2, "rationale", rationale_type, type_=str)- rationale_type = rationale_type or DEFAULT_RATIONALE_TYPE-- extended_kwargs1 = {key: signature.kwargs[key] for key in keys}- extended_kwargs1.update({'rationale': rationale_type, last_key: signature.kwargs[last_key]})-- extended_kwargs2 = {key: signature.kwargs[key] for key in keys}- extended_kwargs2.update({'hint': DEFAULT_HINT_TYPE, 'rationale': rationale_type, last_key: signature.kwargs[last_key]})-- self.extended_signature1 = dsp.Template(signature.instructions, **extended_kwargs1)- self.extended_signature2 = dsp.Template(signature.instructions, **extended_kwargs2)+ DEFAULT_HINT_TYPE = dspy.OutputField()+ self.extended_signature2 = self.extended_signature1.insert(-2, "hint", DEFAULT_HINT_TYPE, type_=str)def forward(self, **kwargs):signature = self.signaturediff --git a/dspy/predict/langchain.py b/dspy/predict/langchain.pyindex e3ddd37ce..4be855e8d 100644--- a/dspy/predict/langchain.py+++ b/dspy/predict/langchain.py@@ -13,6 +13,8 @@ from dspy.signatures.signature import infer_prefixfrom langchain_core.pydantic_v1 import Extrafrom langchain_core.runnables import Runnable+# TODO: This class is currently hard to test, because it hardcodes gpt-4 usage:+# gpt4T = dspy.OpenAI(model='gpt-4-1106-preview', max_tokens=4000, model_type='chat')class Template2Signature(dspy.Signature):"""You are a processor for prompts. I will give you a prompt template (Python f-string) for an arbitrary task for other LMs.diff --git a/dspy/predict/multi_chain_comparison.py b/dspy/predict/multi_chain_comparison.pyindex 99c2b43c5..89fc73297 100644--- a/dspy/predict/multi_chain_comparison.py+++ b/dspy/predict/multi_chain_comparison.py@@ -1,38 +1,55 @@+import dspy+from dspy.signatures.signature import ensure_signaturefrom .predict import Predictfrom ..primitives.program import Moduleimport dsp+class MultiChainComparison(Module):def __init__(self, signature, M=3, temperature=0.7, **config):super().__init__()self.M = M- signature = Predict(signature).signature- *keys, last_key = signature.kwargs.keys()+ signature = ensure_signature(signature)- extended_kwargs = {key: signature.kwargs[key] for key in keys}+ *_, self.last_key = signature.output_fields.keys()for idx in range(M):- candidate_type = dsp.Type(prefix=f"Student Attempt #{idx+1}:", desc="${reasoning attempt}")- extended_kwargs.update({f'reasoning_attempt_{idx+1}': candidate_type})-- rationale_type = dsp.Type(prefix="Accurate Reasoning: Thank you everyone. Let's now holistically", desc="${corrected reasoning}")- extended_kwargs.update({'rationale': rationale_type, last_key: signature.kwargs[last_key]})+ signature = signature.append(+ f"reasoning_attempt_{idx+1}",+ dspy.InputField(+ prefix=f"Student Attempt #{idx+1}:", desc="${reasoning attempt}"+ ),+ )++ signature = signature.prepend(+ "rationale",+ dspy.OutputField(+ prefix="Accurate Reasoning: Thank you everyone. Let's now holistically",+ desc="${corrected reasoning}",+ ),+ )- signature = dsp.Template(signature.instructions, **extended_kwargs)self.predict = Predict(signature, temperature=temperature, **config)- self.last_key = last_key-+def forward(self, completions, **kwargs):attempts = []for c in completions:- rationale = c.rationale.strip().split('\n')[0].strip()- answer = c[self.last_key].strip().split('\n')[0].strip()- attempts.append(f"«I'm trying to {rationale} I'm not sure but my prediction is {answer}»")+ rationale = c.rationale.strip().split("\n")[0].strip()+ answer = c[self.last_key].strip().split("\n")[0].strip()+ attempts.append(+ f"«I'm trying to {rationale} I'm not sure but my prediction is {answer}»"+ )assert len(attempts) == self.M, len(attempts)- kwargs = {**{f'reasoning_attempt_{idx+1}': attempt for idx, attempt in enumerate(attempts)}, **kwargs}+ kwargs = {+ **{+ f"reasoning_attempt_{idx+1}": attempt+ for idx, attempt in enumerate(attempts)+ },+ **kwargs,+ }return self.predict(**kwargs)diff --git a/dspy/predict/predict.py b/dspy/predict/predict.pyindex c68ac1a5a..3823a72a7 100644--- a/dspy/predict/predict.py+++ b/dspy/predict/predict.py@@ -3,42 +3,16 @@ import randomfrom dspy.predict.parameter import Parameterfrom dspy.primitives.prediction import Prediction-from dspy.signatures.field import InputField, OutputField-from dspy.signatures.signature import infer_prefix+from dspy.signatures.signature import ensure_signature, signature_to_templateclass Predict(Parameter):def __init__(self, signature, **config):self.stage = random.randbytes(8).hex()- self.signature = signature #.signature+ self.signature = ensure_signature(signature)self.config = configself.reset()- # if the signature is a string- if isinstance(signature, str):- inputs, outputs = signature.split("->")- inputs, outputs = inputs.split(","), outputs.split(",")- inputs, outputs = [field.strip() for field in inputs], [field.strip() for field in outputs]-- assert all(len(field.split()) == 1 for field in (inputs + outputs))-- inputs_ = ', '.join([f"`{field}`" for field in inputs])- outputs_ = ', '.join([f"`{field}`" for field in outputs])-- instructions = f"""Given the fields {inputs_}, produce the fields {outputs_}."""-- inputs = {k: InputField() for k in inputs}- outputs = {k: OutputField() for k in outputs}-- for k, v in inputs.items():- v.finalize(k, infer_prefix(k))-- for k, v in outputs.items():- v.finalize(k, infer_prefix(k))-- self.signature = dsp.Template(instructions, **inputs, **outputs)--def reset(self):self.lm = Noneself.traces = []@@ -51,43 +25,47 @@ class Predict(Parameter):# Cache the signature instructions and the last field's name.state["signature_instructions"] = self.signature.instructions- state["signature_prefix"] = self.signature.fields[-1].name++ *_, last_key = self.signature.fields.keys()+ state["signature_prefix"] = self.signature.fields[last_key].json_schema_extra['prefix']return statedef load_state(self, state):for name, value in state.items():setattr(self, name, value)-+# Reconstruct the signature.if "signature_instructions" in state:instructions = state["signature_instructions"]- self.signature.instructions = instructions-+ self.signature = self.signature.with_instructions(instructions)+if "signature_prefix" in state:prefix = state["signature_prefix"]- self.signature.fields[-1] = self.signature.fields[-1]._replace(name=prefix)-+ *_, last_key = self.signature.fields.keys()+ self.signature = self.signature.with_updated_fields(last_key, prefix=prefix)+def __call__(self, **kwargs):return self.forward(**kwargs)-+def forward(self, **kwargs):# Extract the three privileged keyword arguments.- new_signature = kwargs.pop("new_signature", None)- signature = kwargs.pop("signature", self.signature)+ new_signature = ensure_signature(kwargs.pop("new_signature", None))+ signature = ensure_signature(kwargs.pop("signature", self.signature))demos = kwargs.pop("demos", self.demos)config = dict(**self.config, **kwargs.pop("config", {}))# Get the right LM to use.lm = kwargs.pop("lm", self.lm) or dsp.settings.lm+ assert lm is not None, "No LM is loaded."# If temperature is 0.0 but its n > 1, set temperature to 0.7.temperature = config.get("temperature", None)- temperature = lm.kwargs['temperature'] if temperature is None else temperature+ temperature = lm.kwargs["temperature"] if temperature is None else temperaturenum_generations = config.get("n", None)if num_generations is None:- num_generations = lm.kwargs.get('n', lm.kwargs.get('num_generations', None))+ num_generations = lm.kwargs.get("n", lm.kwargs.get("num_generations", None))if (temperature is None or temperature <= 0.15) and num_generations > 1:config["temperature"] = 0.7@@ -98,25 +76,35 @@ class Predict(Parameter):x = dsp.Example(demos=demos, **kwargs)if new_signature is not None:- signature = dsp.Template(signature.instructions, **new_signature)+ signature = new_signature++ assert all(k in kwargs for k in signature.input_fields), "Not all input fields were provided."++ # Switch to legacy format for dsp.generate+ template = signature_to_template(signature)if self.lm is None:- x, C = dsp.generate(signature, **config)(x, stage=self.stage)+ x, C = dsp.generate(template, **config)(x, stage=self.stage)else:+ # Note: query_only=True means the instructions and examples are not included.+ # I'm not really sure why we'd want to do that, but it's there.with dsp.settings.context(lm=self.lm, query_only=True):- # print(f"using lm = {self.lm} !")- x, C = dsp.generate(signature, **config)(x, stage=self.stage)+ x, C = dsp.generate(template, **config)(x, stage=self.stage)++ assert self.stage in x, "The generated (input, output) example was not stored"completions = []for c in C:completions.append({})- for field in signature.fields:+ for field in template.fields:if field.output_variable not in kwargs.keys():- completions[-1][field.output_variable] = getattr(c, field.output_variable)+ completions[-1][field.output_variable] = getattr(+ c, field.output_variable+ )pred = Prediction.from_completions(completions, signature=signature)-+if kwargs.pop("_trace", True) and dsp.settings.trace is not None:trace = dsp.settings.tracetrace.append((self, {**kwargs}, pred))@@ -125,7 +113,7 @@ class Predict(Parameter):def update_config(self, **kwargs):self.config = {**self.config, **kwargs}-+def get_config(self):return self.config@@ -133,7 +121,6 @@ class Predict(Parameter):return f"{self.__class__.__name__}({self.signature})"-# TODO: get some defaults during init from the context window?# # TODO: FIXME: Hmm, I guess expected behavior is that contexts can# affect execution. Well, we need to determine whether context dominates, __init__ demoninates, or forward dominates.diff --git a/dspy/predict/program_of_thought.py b/dspy/predict/program_of_thought.pyindex 65d1613b3..516c1129c 100644--- a/dspy/predict/program_of_thought.py+++ b/dspy/predict/program_of_thought.py@@ -1,94 +1,159 @@import dspimport dspy+from dspy.signatures.signature import ensure_signaturefrom ..primitives.program import Modulefrom ..primitives.python_interpreter import CodePrompt, PythonInterpreterimport re+class ProgramOfThought(Module):def __init__(self, signature, max_iters=3):super().__init__()- self.signature = signature = dspy.Predict(signature).signature+ self.signature = signature = ensure_signature(signature)self.max_iters = max_iters- self.input_fields = signature.input_fields()- self.output_fields = signature.output_fields()+ self.input_fields = signature.input_fields+ self.output_fields = signature.output_fields- inputs_ = ', '.join([f"`{field_name}`" for field_name in self.input_fields.keys()])- outputs_ = ', '.join([f"`{field_name}`" for field_name in self.output_fields.keys()])+ inputs_ = ", ".join(+ [f"`{field_name}`" for field_name in self.input_fields.keys()]+ )+ outputs_ = ", ".join(+ [f"`{field_name}`" for field_name in self.output_fields.keys()]+ )assert len(self.output_fields) == 1, "PoT only supports one output field."-+instr = []- instr.append(f"You will be given {inputs_} and you will respond with {outputs_}.")- instr.append(f"Generating executable Python code that programmatically computes the correct {outputs_}.")- instr.append(f"After you're done with the computation, make sure the last line in your code evaluates to the correct value for {outputs_}.")- instr = '\n'.join(instr)-- self.code_generate = dspy.ChainOfThought(dsp.Template(self._generate_instruction('generate'), **self._generate_signature('generate')))- self.code_regenerate = dspy.ChainOfThought(dsp.Template(self._generate_instruction('regenerate'), **self._generate_signature('regenerate')))- self.generate_answer = dspy.ChainOfThought(dsp.Template(self._generate_instruction('answer'), **self._generate_signature('answer')))+ instr.append(+ f"You will be given {inputs_} and you will respond with {outputs_}."+ )+ instr.append(+ f"Generating executable Python code that programmatically computes the correct {outputs_}."+ )+ instr.append(+ f"After you're done with the computation, make sure the last line in your code evaluates to the correct value for {outputs_}."+ )+ instr = "\n".join(instr)++ self.code_generate = dspy.ChainOfThought(+ dspy.Signature(+ self._generate_signature("generate").fields,+ self._generate_instruction("generate"),+ )+ )+ self.code_regenerate = dspy.ChainOfThought(+ dspy.Signature(+ self._generate_signature("regenerate").fields,+ self._generate_instruction("regenerate"),+ )+ )+ self.generate_answer = dspy.ChainOfThought(+ dspy.Signature(+ self._generate_signature("answer").fields,+ self._generate_instruction("answer"),+ )+ )def _generate_signature(self, mode):signature_dict = dict(self.input_fields)fields_for_mode = {- 'generate': {- 'generated_code': dspy.OutputField(prefix="Code:", desc="python code that answers the question", format=str)+ "generate": {+ "generated_code": dspy.OutputField(+ prefix="Code:",+ desc="python code that answers the question",+ format=str,+ )+ },+ "regenerate": {+ "previous_code": dspy.InputField(+ prefix="Previous Code:",+ desc="previously-generated python code that errored",+ format=str,+ ),+ "error": dspy.InputField(+ prefix="Error:",+ desc="error message from previously-generated python code",+ ),+ "generated_code": dspy.OutputField(+ prefix="Code:",+ desc="python code that answers the question",+ format=str,+ ),},- 'regenerate': {- 'previous_code': dspy.InputField(prefix="Previous Code:", desc="previously-generated python code that errored", format=str),- 'error': dspy.InputField(prefix="Error:", desc="error message from previously-generated python code"),- 'generated_code': dspy.OutputField(prefix="Code:", desc="python code that answers the question", format=str)+ "answer": {+ "final_generated_code": dspy.InputField(+ prefix="Code:",+ desc="python code that answers the question",+ format=str,+ ),+ "code_output": dspy.InputField(+ prefix="Code Output:",+ desc="output of previously-generated python code",+ ),+ "answer": self.signature.fields["answer"],},- 'answer': {- 'final_generated_code': dspy.InputField(prefix="Code:", desc="python code that answers the question", format=str),- 'code_output': dspy.InputField(prefix="Code Output:", desc="output of previously-generated python code"),- 'answer': self.signature.kwargs["answer"]- }}signature_dict.update(fields_for_mode[mode])- return signature_dict+ return dspy.Signature(signature_dict)def _generate_instruction(self, mode):- mode_inputs = ', '.join([f"`{field_name}`" for field_name in self._generate_signature(mode).keys() if isinstance(self._generate_signature(mode)[field_name], dspy.InputField)])- mode_outputs = ', '.join([f"`{field_name}`" for field_name in self._generate_signature(mode).keys() if isinstance(self._generate_signature(mode)[field_name], dspy.OutputField)])- if mode == 'generate':+ mode_inputs = ", ".join(+ [+ f"`{field_name}`"+ for field_name in self._generate_signature(mode).input_fields+ ]+ )+ mode_outputs = ", ".join(+ [+ f"`{field_name}`"+ for field_name in self._generate_signature(mode).output_fields+ ]+ )+ if mode == "generate":instr = [f"You will be given {mode_inputs} and you will respond with {mode_outputs}.",f"Generating executable Python code that programmatically computes the correct {mode_outputs}.",- f"After you're done with the computation, make sure the last line in your code evaluates to the correct value for {mode_outputs}."+ f"After you're done with the computation, make sure the last line in your code evaluates to the correct value for {mode_outputs}.",]- elif mode == 'regenerate':+ elif mode == "regenerate":instr = [f"You are given {mode_inputs} due to an error in previous code.",- f"Your task is to correct the error and provide the new {mode_outputs}."+ f"Your task is to correct the error and provide the new {mode_outputs}.",]else: # mode == 'answer'instr = [f"Given the final code {mode_inputs}, provide the final {mode_outputs}."]- return '\n'.join(instr)+ return "\n".join(instr)def parse_code(self, code_data):- code = code_data.get('generated_code', '').split('---', 1)[0].split('\n\n\n', 1)[0]- code_match = re.search(r'```python[ \n](.*?)[ \n]```?', code, re.DOTALL)- code_block = (code_match.group(1) if code_match else code).replace('\\n', '\n')+ code = (+ code_data.get("generated_code", "").split("---", 1)[0].split("\n\n\n", 1)[0]+ )+ code_match = re.search(r"```python[ \n](.*?)[ \n]```?", code, re.DOTALL)+ code_block = (code_match.group(1) if code_match else code).replace("\\n", "\n")if not code_block:return code, "Error: Empty code after parsing."- if "\n" not in code_block and code_block.count('=') > 1:+ if "\n" not in code_block and code_block.count("=") > 1:return code, "Error: Code format is not correct."- lines = code_block.split('\n')- last_line_match = re.match(r'^(\w+)\s*=', lines[-1].strip())+ lines = code_block.split("\n")+ last_line_match = re.match(r"^(\w+)\s*=", lines[-1].strip())if last_line_match and len(lines) > 1:- code_block += '\n' + last_line_match.group(1)+ code_block += "\n" + last_line_match.group(1)else:- code_block = re.sub(r'([a-zA-Z_]\w* *=.*?)(?=[a-zA-Z_]\w* *=)', r'\1\n', code_block)- code_block = re.sub(r'([a-zA-Z_]\w* *=.*?)([a-zA-Z_]\w*)$', r'\1\n\2', code_block)+ code_block = re.sub(+ r"([a-zA-Z_]\w* *=.*?)(?=[a-zA-Z_]\w* *=)", r"\1\n", code_block+ )+ code_block = re.sub(+ r"([a-zA-Z_]\w* *=.*?)([a-zA-Z_]\w*)$", r"\1\n\2", code_block+ )return code_block, Nonedef execute_code(self, code):if not code:- return code, None, 'Error: Empty code before execution.'+ return code, None, "Error: Empty code before execution."code_prompt = CodePrompt(code, code_type="python")interpreter = PythonInterpreter(action_space={"print": print})try:@@ -96,19 +161,26 @@ class ProgramOfThought(Module):return code, output, Noneexcept Exception as e:return code, None, str(e)-+def forward(self, **kwargs):code_data = self.code_generate(question=kwargs["question"])parsed_code, error = self.parse_code(code_data)+ # FIXME: Don't try to execute the code if it didn't parsecode, output, error = self.execute_code(parsed_code)hop = 0while hop < self.max_iters and error:- print('Error in code execution')- code_data = self.code_regenerate(question=kwargs["question"], previous_code=code, error=error)+ print("Error in code execution")+ code_data = self.code_regenerate(+ question=kwargs["question"], previous_code=code, error=error+ )parsed_code, error = self.parse_code(code_data)+ # FIXME: Don't try to execute the code if it didn't parse+ code, output, error = self.execute_code(parsed_code)hop += 1if hop == self.max_iters:- print('Max hops reached. Error persists.')+ print("Max hops reached. Error persists.")return None- answer_gen_result = self.generate_answer(question=kwargs["question"], final_generated_code=code, code_output=output)+ answer_gen_result = self.generate_answer(+ question=kwargs["question"], final_generated_code=code, code_output=output+ )return answer_gen_resultdiff --git a/dspy/predict/react.py b/dspy/predict/react.pyindex 7dc3d1bd9..ef24c6aca 100644--- a/dspy/predict/react.py+++ b/dspy/predict/react.py@@ -1,5 +1,6 @@import dspimport dspy+from dspy.signatures.signature import ensure_signaturefrom ..primitives.program import Modulefrom .predict import Predict@@ -10,35 +11,43 @@ from .predict import Predictclass ReAct(Module):def __init__(self, signature, max_iters=5, num_results=3, tools=None):super().__init__()- self.signature = signature = dspy.Predict(signature).signature+ self.signature = signature = ensure_signature(signature)self.max_iters = max_itersself.tools = tools or [dspy.Retrieve(k=num_results)]- self.tools = {tool.name: tool for tool in self.tools} #if isinstance(self.tools, list) else self.tools+ self.tools = {tool.name: tool for tool in self.tools}- self.input_fields = {k: v for k, v in self.signature.kwargs.items() if isinstance(v, dspy.InputField)}- self.output_fields = {k: v for k, v in self.signature.kwargs.items() if isinstance(v, dspy.OutputField)}+ self.input_fields = self.signature.input_fields+ self.output_fields = self.signature.output_fields- inputs, outputs = signature.fields[:-1], signature.fields[-1:]+ assert len(self.output_fields) == 1, "ReAct only supports one output field."- inputs_ = ', '.join([f"`{field.input_variable}`" for field in inputs])- outputs_ = ', '.join([f"`{field.output_variable}`" for field in outputs])+ inputs_ = ", ".join([f"`{k}`" for k in self.input_fields.keys()])+ outputs_ = ", ".join([f"`{k}`" for k in self.output_fields.keys()])- assert len(outputs) == 1, "ReAct only supports one output field."+ instr = [+ f"You will be given {inputs_} and you will respond with {outputs_}.\n",+ "To do this, you will interleave Thought, Action, and Observation steps.\n",+ "Thought can reason about the current situation, and Action can be the following types:\n",+ ]- instr = []- instr.append(f"You will be given {inputs_} and you will respond with {outputs_}.\n")- instr.append("To do this, you will interleave Thought, Action, and Observation steps.\n")- instr.append("Thought can reason about the current situation, and Action can be the following types:\n")-- self.tools['Finish'] = dspy.Example(name="Finish", input_variable=outputs_.strip('`'), desc=f"returns the final {outputs_} and finishes the task")+ self.tools["Finish"] = dspy.Example(+ name="Finish",+ input_variable=outputs_.strip("`"),+ desc=f"returns the final {outputs_} and finishes the task",+ )for idx, tool in enumerate(self.tools):tool = self.tools[tool]- instr.append(f"({idx+1}) {tool.name}[{tool.input_variable}], which {tool.desc}")-- instr = '\n'.join(instr)- self.react = [Predict(dsp.Template(instr, **self._generate_signature(i))) for i in range(1, max_iters + 1)]+ instr.append(+ f"({idx+1}) {tool.name}[{tool.input_variable}], which {tool.desc}"+ )++ instr = "\n".join(instr)+ self.react = [+ Predict(dspy.Signature(self._generate_signature(i), instr))+ for i in range(1, max_iters + 1)+ ]def _generate_signature(self, iters):signature_dict = {}@@ -46,25 +55,42 @@ class ReAct(Module):signature_dict[key] = valfor j in range(1, iters + 1):- signature_dict[f"Thought_{j}"] = dspy.OutputField(prefix=f"Thought {j}:", desc="next steps to take based on last observation")-- tool_list = ' or '.join([f"{tool.name}[{tool.input_variable}]" for tool in self.tools.values() if tool.name != 'Finish'])- signature_dict[f"Action_{j}"] = dspy.OutputField(prefix=f"Action {j}:", desc=f"always either {tool_list} or, when done, Finish[answer]")+ signature_dict[f"Thought_{j}"] = dspy.OutputField(+ prefix=f"Thought {j}:",+ desc="next steps to take based on last observation",+ )++ tool_list = " or ".join(+ [+ f"{tool.name}[{tool.input_variable}]"+ for tool in self.tools.values()+ if tool.name != "Finish"+ ]+ )+ signature_dict[f"Action_{j}"] = dspy.OutputField(+ prefix=f"Action {j}:",+ desc=f"always either {tool_list} or, when done, Finish[answer]",+ )if j < iters:- signature_dict[f"Observation_{j}"] = dspy.OutputField(prefix=f"Observation {j}:", desc="observations based on action", format=dsp.passages2text)+ signature_dict[f"Observation_{j}"] = dspy.OutputField(+ prefix=f"Observation {j}:",+ desc="observations based on action",+ format=dsp.passages2text,+ )return signature_dict-+def act(self, output, hop):try:action = output[f"Action_{hop+1}"]- action_name, action_val = action.strip().split('\n')[0].split('[', 1)- action_val = action_val.rsplit(']', 1)[0]+ action_name, action_val = action.strip().split("\n")[0].split("[", 1)+ action_val = action_val.rsplit("]", 1)[0]- if action_name == 'Finish': return action_val+ if action_name == "Finish":+ return action_val- try:+ try:output[f"Observation_{hop+1}"] = self.tools[action_name](action_val).passagesexcept AttributeError:# Handle the case where 'passages' attribute is missing@@ -72,8 +98,10 @@ class ReAct(Module):output[f"Observation_{hop+1}"] = self.tools[action_name](action_val)except Exception as e:- output[f"Observation_{hop+1}"] = "Failed to parse action. Bad formatting or incorrect action name."-+ output[f"Observation_{hop+1}"] = (+ "Failed to parse action. Bad formatting or incorrect action name."+ )+ raise edef forward(self, **kwargs):args = {key: kwargs[key] for key in self.input_fields.keys() if key in kwargs}@@ -81,9 +109,10 @@ class ReAct(Module):for hop in range(self.max_iters):# with dspy.settings.context(show_guidelines=(i <= 2)):output = self.react[hop](**args)-- if action_val := self.act(output, hop): break++ if action_val := self.act(output, hop):+ breakargs.update(output)# assumes only 1 output field for now - TODO: handling for multiple output fields- return dspy.Prediction(**{list(self.output_fields.keys())[0]: action_val or ''})+ return dspy.Prediction(**{list(self.output_fields.keys())[0]: action_val or ""})diff --git a/dspy/predict/retry.py b/dspy/predict/retry.pyindex b8f06633b..af1d37f98 100644--- a/dspy/predict/retry.py+++ b/dspy/predict/retry.py@@ -9,41 +9,36 @@ class Retry(Predict):def __init__(self, module):super().__init__(module.signature)self.module = module- self.original_signature = module.signature.signature+ self.original_signature = module.signatureself.original_forward = module.forwardself.new_signature = self._create_new_signature(self.original_signature)- def _create_new_signature(self, original_signature):- extended_signature = {}- input_fields = original_signature.input_fields()- output_fields = original_signature.output_fields()- modified_output_fields = {}-- for key, value in output_fields.items():- modified_output_fields[f"past_{key}"] = dspy.InputField(- prefix="Past " + value.prefix,+ def _create_new_signature(self, signature):+ # Add "Past" input fields for each output field+ for key, value in signature.output_fields.items():+ signature = signature.append(f"past_{key}", dspy.InputField(+ prefix="Past " + value.json_schema_extra["prefix"],desc="past output with errors",- format=value.format,- )-- extended_signature.update(input_fields)- extended_signature.update(modified_output_fields)+ format=value.json_schema_extra.get("format"),+ ))- extended_signature["feedback"] = dspy.InputField(+ signature = signature.append("feedback", dspy.InputField(prefix="Instructions:",desc="Some instructions you must satisfy",format=str,- )- extended_signature.update(output_fields)+ ))- return extended_signature+ return signature- def forward(self, *args, **kwargs):- for key, value in kwargs["past_outputs"].items():+ def forward(self, *, past_outputs, **kwargs):+ # Convert the dict past_outputs={"answer": ...} to kwargs+ # {past_answer=..., ...}+ for key, value in past_outputs.items():past_key = f"past_{key}"- if past_key in self.new_signature:+ if past_key in self.new_signature.input_fields:kwargs[past_key] = value- del kwargs["past_outputs"]+ # Tell the wrapped module to use the new signature.+ # Note: This only works if the wrapped module is a Predict or ChainOfThought.kwargs["new_signature"] = self.new_signaturereturn self.original_forward(**kwargs)diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 5f89896e2..feb54005c 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -238,7 +238,6 @@ def backtrack_handler(func, bypass_suggest=True, max_backtracks=2):else:try:dsp.settings.trace.clear()- # print("backtrack", dspy.settings.backtrack_to)result = func(*args, **kwargs)breakexcept (DSPySuggestionError, DSPyAssertionError) as e:@@ -282,13 +281,13 @@ def backtrack_handler(func, bypass_suggest=True, max_backtracks=2):dspy.settings.backtrack_to].append(error_msg)- output_fields = vars(error_state[0].signature.signature)+ # assert isinstance(error_state[0].signature, dspy.Signature)+ output_fields = error_state[0].signature.output_fieldspast_outputs = {}- for field_name, field_obj in output_fields.items():- if isinstance(field_obj, dspy.OutputField):- past_outputs[field_name] = getattr(- error_state[2], field_name, None- )+ for field_name in output_fields.keys():+ past_outputs[field_name] = getattr(+ error_state[2], field_name, None+ )# save latest failure trace for predictor per suggestionerror_ip = error_state[1]diff --git a/dspy/primitives/python_interpreter.py b/dspy/primitives/python_interpreter.pyindex f05ec0111..11ae2795a 100644--- a/dspy/primitives/python_interpreter.py+++ b/dspy/primitives/python_interpreter.py@@ -14,6 +14,7 @@import astimport difflibimport importlib+import reimport typingimport inspectfrom typing import (@@ -506,10 +507,11 @@ class TextPrompt(str):@propertydef key_words(self) -> Set[str]:- r"""Returns a set of strings representing the keywords in the prompt.- """- from camel.utils import get_prompt_template_key_words- return get_prompt_template_key_words(self)+ """Returns a set of strings representing the keywords in the prompt."""+ # Regex to find format placeholders within the string, excluding escaped braces+ pattern = re.compile(r"\{([^{}]+)\}")+ found = pattern.findall(self)+ return set(found)def format(self, *args: Any, **kwargs: Any) -> 'TextPrompt':r"""Overrides the built-in :obj:`str.format` method to allow fordiff --git a/dspy/signatures/field.py b/dspy/signatures/field.pyindex 848439b6d..7822c625c 100644--- a/dspy/signatures/field.py+++ b/dspy/signatures/field.py@@ -1,31 +1,71 @@-import re-import dsp+import pydantic-class Field:++def move_kwargs(**kwargs):+ # Pydantic doesn't allow arbitrary arguments to be given to fields,+ # but asks that+ # > any extra data you want to add to the JSON schema should be passed+ # > as a dictionary to the json_schema_extra keyword argument.+ # See: https://docs.pydantic.dev/2.6/migration/#changes-to-pydanticfield+ pydantic_kwargs = {}+ json_schema_extra = {}+ for k, v in kwargs.items():+ if k in ["desc", "prefix", "format", "parser", "__dspy_field_type"]:+ json_schema_extra[k] = v+ else:+ pydantic_kwargs[k] = v+ pydantic_kwargs["json_schema_extra"] = json_schema_extra+ return pydantic_kwargs+++def InputField(**kwargs):+ return pydantic.Field(**move_kwargs(**kwargs, __dspy_field_type="input"))+++def OutputField(**kwargs):+ return pydantic.Field(**move_kwargs(**kwargs, __dspy_field_type="output"))+++def new_to_old_field(field):+ return (+ OldInputField+ if field.json_schema_extra["__dspy_field_type"] == "input"+ else OldOutputField+ )(+ prefix=field.json_schema_extra["prefix"],+ desc=field.json_schema_extra["desc"],+ format=field.json_schema_extra.get("format"),+ )+++class OldField:"""A more ergonomic datatype that infers prefix and desc if omitted."""+def __init__(self, *, prefix=None, desc=None, input, format=None):self.prefix = prefix # This can be None initially and set laterself.desc = descself.format = format-+def finalize(self, key, inferred_prefix):"""Set the prefix if it's not provided explicitly."""if self.prefix is None:self.prefix = inferred_prefix + ":"-+if self.desc is None:- self.desc = f'${{{key}}}'-+ self.desc = f"${{{key}}}"+def __repr__(self):return f"{self.__class__.__name__}(prefix={self.prefix}, desc={self.desc})"-+def __eq__(self, __value: object) -> bool:return self.__dict__ == __value.__dict__-class InputField(Field):++class OldInputField(OldField):def __init__(self, *, prefix=None, desc=None, format=None):super().__init__(prefix=prefix, desc=desc, input=True, format=format)-class OutputField(Field):++class OldOutputField(OldField):def __init__(self, *, prefix=None, desc=None, format=None):super().__init__(prefix=prefix, desc=desc, input=False, format=format)diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.pyindex ea31a46d4..b73c956d7 100644--- a/dspy/signatures/signature.py+++ b/dspy/signatures/signature.py@@ -1,176 +1,256 @@-import re+from copy import deepcopyimport dsp+from pydantic import BaseModel, Field, create_model+from typing import Type, Union, Dict, Tuple+import re-from .field import Field, InputField, OutputField-import threading+from dspy.signatures.field import InputField, OutputField, new_to_old_field-class SignatureMeta(type):- _thread_local_storage = threading.local()- class _SignatureNamespace:- def __init__(self, fields):- for key, value in fields.items():- setattr(self, key, value)+def signature_to_template(signature):+ """Convert from new to legacy format"""+ return dsp.Template(+ signature.instructions,+ **{name: new_to_old_field(field) for name, field in signature.fields.items()},+ )- def input_fields(self):- return {k: v for k, v in self.__dict__.items() if isinstance(v, InputField)}- def output_fields(self):- return {k: v for k, v in self.__dict__.items() if isinstance(v, OutputField)}-+def _default_instructions(cls):+ inputs_ = ", ".join([f"`{field}`" for field in cls.input_fields.keys()])+ outputs_ = ", ".join([f"`{field}`" for field in cls.output_fields.keys()])+ return f"Given the fields {inputs_}, produce the fields {outputs_}."- def __new__(cls, name, bases, class_dict):- type_attributes = {}- for k, v in list(class_dict.items()):- if isinstance(v, Field):- v.finalize(k, infer_prefix(k))- type_attributes[k] = v- del class_dict[k]+class SignatureMeta(type(BaseModel)):+ def __new__(mcs, name, bases, namespace, **kwargs):+ # Set `str` as the default type for all fields+ raw_annotations = namespace.get("__annotations__", {})+ for name, field in namespace.items():+ if not name.startswith("__") and name not in raw_annotations:+ raw_annotations[name] = str+ namespace["__annotations__"] = raw_annotations- instructions = class_dict.get('__doc__') or ""+ # Let Pydantic do its thing+ cls = super().__new__(mcs, name, bases, namespace, **kwargs)- new_class = super().__new__(cls, name, bases, class_dict)+ if cls.__doc__ is None:+ cls.__doc__ = _default_instructions(cls)- # Attach the _SignatureNamespace directly to the class- setattr(new_class, 'signature', cls._SignatureNamespace(type_attributes))+ # Ensure all fields are declared with InputField or OutputField+ cls._validate_fields()- # Create and attach the template directly to the class- setattr(new_class, '_template', dsp.Template(instructions=instructions, **type_attributes))+ # Ensure all fields have a prefix+ for name, field in cls.model_fields.items():+ if "prefix" not in field.json_schema_extra:+ field.json_schema_extra["prefix"] = infer_prefix(name) + ":"+ if "desc" not in field.json_schema_extra:+ field.json_schema_extra["desc"] = f"${{{name}}}"- return new_class+ return cls++ def _validate_fields(cls):+ for name, field in cls.model_fields.items():+ extra = field.json_schema_extra or {}+ field_type = extra.get("__dspy_field_type")+ if field_type not in ["input", "output"]:+ raise TypeError(+ f"Field '{name}' in '{cls.__name__}' must be declared with InputField or OutputField."+ )@property- def kwargs(cls):- return cls.signature.fields-- def __call__(cls, *args, **kwargs):- if len(args) == 1 and isinstance(args[0], str):- instance = super(SignatureMeta, cls).__call__(*args, **kwargs)- return instance- #old- return cls._template(*args, **kwargs)-- def __getattr__(cls, attr):- # Redirect attribute access to the template object when accessed on the class directly- if attr not in cls.__dict__:- return getattr(cls._template, attr)- return super().__getattr__(attr)--class Signature(metaclass=SignatureMeta):- def __init__(self, signature: str = "", instructions: str = ""):- self.signature = signature- self.instructions = instructions- self.fields = {}- self.parse_structure()-- def __getattr__(self, attr):- if attr not in self.__dict__:- return getattr(self.__class__, attr)- return super().__getattr__(attr)+ def signature(cls) -> str:+ in_args = ", ".join(cls.input_fields.keys())+ out_args = ", ".join(cls.output_fields.keys())+ return f"{in_args} -> {out_args}"@property- def kwargs(self):- return {k: v for k, v in self.fields.items()}-- def parse_structure(self):- inputs_str, outputs_str = self.signature.split("->")- for name in inputs_str.split(","):- self.add_field(name.strip(), InputField())- for name in outputs_str.split(","):- self.add_field(name.strip(), OutputField())-- def attach(self, **kwargs):- for key, (prefix, desc) in kwargs.items():- field_type = self.fields.get(key)- if not field_type:- raise ValueError(f"{key} does not exist in this signature")- field_map = {- InputField: InputField(prefix=prefix, desc=desc),- OutputField: OutputField(prefix=prefix, desc=desc)- }- self.fields[key] = field_map.get(type(field_type))- return self-- def add_field(self, field_name: str, field_type, position="append"):- if field_name in self.fields:- raise ValueError(f"{field_name} already exists in fields.")- if isinstance(field_type, (InputField, OutputField)):- field_instance = field_type- else:- raise ValueError(f"non-existent {field_type}.")- if isinstance(field_instance, InputField) and position == "append":- input_fields = self.input_fields()- if input_fields:- last_input_key = list(input_fields.keys())[-1]- index = list(self.fields.keys()).index(last_input_key) + 1- self.fields = {**dict(list(self.fields.items())[:index]), field_name: field_instance, **dict(list(self.fields.items())[index:])}- else:- self.fields[field_name] = field_instance- elif isinstance(field_instance, OutputField) and position == "prepend":- output_fields = self.output_fields()- if output_fields:- first_output_key = list(output_fields.keys())[0]- index = list(self.fields.keys()).index(first_output_key)- self.fields = {**dict(list(self.fields.items())[:index]), field_name: field_instance, **dict(list(self.fields.items())[index:])}- else:- self.fields[field_name] = field_instance- elif position == "prepend":- self.fields = {field_name: field_instance, **self.fields}- elif position == "append":- self.fields[field_name] = field_instance+ def instructions(cls) -> str:+ return getattr(cls, "__doc__", "")++ def with_instructions(cls, instructions: str):+ return create_model(+ cls.__name__, __base__=Signature, __doc__=instructions, **cls.fields+ )++ @property+ def fields(cls):+ # Make sure to give input fields before output fields+ return {**cls.input_fields, **cls.output_fields}++ def with_updated_fields(cls, name, **kwargs):+ """Returns a new Signature type with the field, name, updated+ with fields[name].json_schema_extra[key] = value."""+ fields_copy = deepcopy(cls.fields)+ fields_copy[name].json_schema_extra = {+ **fields_copy[name].json_schema_extra,+ **kwargs,+ }+ return create_model(+ cls.__name__, __base__=Signature, __doc__=cls.instructions, **fields_copy+ )++ @property+ def input_fields(cls):+ return cls._get_fields_with_type("input")++ @property+ def output_fields(cls):+ return cls._get_fields_with_type("output")++ def _get_fields_with_type(cls, field_type):+ return {+ k: v+ for k, v in cls.model_fields.items()+ if v.json_schema_extra["__dspy_field_type"] == field_type+ }++ def prepend(cls, name, field, type_=None):+ return cls.insert(0, name, field, type_)++ def append(cls, name, field, type_=None):+ return cls.insert(-1, name, field, type_)++ def insert(cls, index: int, name: str, field, type_: Type = None):+ # It's posisble to set the type as annotation=type in pydantic.Field(...)+ # But this may be annoying for users, so we allow them to pass the type+ if type_ is not None:+ field.annotation = type_++ input_fields = list(cls.input_fields.items())+ output_fields = list(cls.output_fields.items())++ # Choose the list to insert into based on the field type+ lst = (+ input_fields+ if field.json_schema_extra["__dspy_field_type"] == "input"+ else output_fields+ )+ # We support negative insert indices+ if index < 0:+ index += len(lst) + 1+ if index < 0 or index > len(lst):+ raise ValueError(f"Invalid index: {index}")+ lst.insert(index, (name, field))++ new_fields = dict(input_fields + output_fields)+ new_signature = create_model(+ cls.__name__ + "'", __base__=Signature, **new_fields+ )+ new_signature.__doc__ = cls.instructions+ return new_signature++ def _parse_signature(cls, signature: str) -> Tuple[Type, Field]:+ pattern = r"^\s*[\w\s,]+\s*->\s*[\w\s,]+\s*$"+ if not re.match(pattern, signature):+ raise ValueError(f"Invalid signature format: '{signature}'")++ fields = {}+ inputs_str, outputs_str = map(str.strip, signature.split("->"))+ inputs = [v.strip() for v in inputs_str.split(",") if v.strip()]+ outputs = [v.strip() for v in outputs_str.split(",") if v.strip()]+ for name in inputs:+ fields[name] = (str, InputField())+ for name in outputs:+ fields[name] = (str, OutputField())++ return fields++ def __call__(+ cls,+ signature: Union[str, Dict[str, Tuple[type, Field]]],+ instructions: str = None,+ ):+ """+ Creates a new Signature type with the given fields and instructions.+ Note:+ Even though we're calling a type, we're not making an instance of the type.+ In general we don't allow instances of Signature types to be made. The call+ syntax is only for your convenience.+ Parameters:+ signature: Format: "input1, input2 -> output1, output2"+ instructions: Optional prompt for the signature.+ """++ if isinstance(signature, str):+ fields = cls._parse_signature(signature)else:- raise ValueError(f"invalid field addition. Please verify that your field name: {field_name}, field_type: {field_type}, and expected position: {position} are correct.")+ fields = signature++ # Default prompt when no instructions are provided+ if instructions is None:+ sig = Signature(signature, "") # Simple way to parse input/output fields+ instructions = _default_instructions(sig)++ signature = create_model("Signature", __base__=Signature, **fields)+ signature.__doc__ = instructions+ return signature- def input_fields(self):- return {k: v for k, v in self.fields.items() if isinstance(v, InputField)}+ def equals(cls, other):+ """Compare the JSON schema of two Pydantic models."""+ if not isinstance(other, type) or not issubclass(other, BaseModel):+ return False+ if cls.instructions != other.instructions:+ return False+ for name in cls.fields.keys() | other.fields.keys():+ if name not in other.fields or name not in cls.fields:+ return False+ # TODO: Should we compare the fields?+ return True- def output_fields(self):- return {k: v for k, v in self.fields.items() if isinstance(v, OutputField)}+ def __repr__(cls):+ """+ Outputs something on the form:+ Signature(question, context -> answer+ question: str = InputField(desc="..."),+ context: List[str] = InputField(desc="..."),+ answer: int = OutputField(desc="..."),+ )+ """+ field_reprs = []+ for name, field in cls.fields.items():+ field_reprs.append(f"{name} = Field({field})")+ field_repr = "\n ".join(field_reprs)+ return (+ f"Signature({cls.signature}\n"+ f" instructions={repr(cls.instructions)}\n"+ f" {field_repr}\n)"+ )- def __repr__(self):- s = []- for name, _ in self.fields.items():- value = getattr(self, name, None)- if value:- s.append(f"- {name} = {value}")- else:- s.append(f"- {name} = [field not attached]")- return f'{self.__class__.__name__}\n' + '\n'.join(s)- def __eq__(self, __value: object) -> bool:- return self._template == __value._template+class Signature(BaseModel, metaclass=SignatureMeta):+ pass+def ensure_signature(signature):+ if signature is None:+ return None+ if isinstance(signature, str):+ return Signature(signature)+ return signature+def infer_prefix(attribute_name: str) -> str:"""Infers a prefix from an attribute name."""-+# Convert camelCase to snake_case, but handle sequences of capital letters properly- s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', attribute_name)- intermediate_name = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1)+ s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", attribute_name)+ intermediate_name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1)# Insert underscores around numbers to ensure spaces in the final output- with_underscores_around_numbers = re.sub('([a-zA-Z])(\d)', r'\1_\2', intermediate_name)- with_underscores_around_numbers = re.sub('(\d)([a-zA-Z])', r'\1_\2', with_underscores_around_numbers)+ with_underscores_around_numbers = re.sub(+ r"([a-zA-Z])(\d)", r"\1_\2", intermediate_name+ )+ with_underscores_around_numbers = re.sub(+ r"(\d)([a-zA-Z])", r"\1_\2", with_underscores_around_numbers+ )# Convert snake_case to 'Proper Title Case', but ensure acronyms are uppercased- words = with_underscores_around_numbers.split('_')+ words = with_underscores_around_numbers.split("_")title_cased_words = []for word in words:if word.isupper():title_cased_words.append(word)else:title_cased_words.append(word.capitalize())-- return ' '.join(title_cased_words)--### Testing the function-assert infer_prefix('someAttributeName42IsCool') == 'Some Attribute Name 42 Is Cool'-assert infer_prefix('version2Update') == 'Version 2 Update'-assert infer_prefix('modelT45Enhanced') == 'Model T 45 Enhanced'-assert infer_prefix('someAttributeName') == 'Some Attribute Name'-assert infer_prefix('some_attribute_name') == 'Some Attribute Name'-assert infer_prefix('URLAddress') == 'URL Address'-assert infer_prefix('isHTTPSecure') == 'Is HTTP Secure'-assert infer_prefix('isHTTPSSecure123') == 'Is HTTPS Secure 123'\ No newline at end of file++ return " ".join(title_cased_words)diff --git a/dspy/teleprompt/bootstrap.py b/dspy/teleprompt/bootstrap.pyindex b885bea0f..c4804d63d 100644--- a/dspy/teleprompt/bootstrap.py+++ b/dspy/teleprompt/bootstrap.py@@ -80,7 +80,7 @@ class BootstrapFewShot(Teleprompter):for (name1, predictor1), (name2, predictor2) in zip(student.named_predictors(), teacher.named_predictors()):assert name1 == name2, "Student and teacher must have the same program structure."- assert predictor1.signature == predictor2.signature, f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"+ assert predictor1.signature.equals(predictor2.signature), f"Student and teacher must have the same signatures. {type(predictor1.signature)} != {type(predictor2.signature)}"assert id(predictor1) != id(predictor2), "Student and teacher must be different objects."name2predictor[name1] = None # dict(student=predictor1, teacher=predictor2)diff --git a/dspy/teleprompt/finetune.py b/dspy/teleprompt/finetune.pyindex 82fdf530d..a56adaecb 100644--- a/dspy/teleprompt/finetune.py+++ b/dspy/teleprompt/finetune.py@@ -7,6 +7,8 @@ import randomimport ujsonfrom datasets.fingerprint import Hasher+from dspy.signatures.signature import signature_to_template+# from dspy.primitives import Examplefrom .teleprompt import Teleprompter@@ -84,8 +86,9 @@ class BootstrapFinetune(Teleprompter):demo = dict(demo)# TODO: FIXME: generalize.- completion = demo.pop(predictor.signature.fields[-1].output_variable)- prompt = predictor.signature.query(dsp.Example(demos=[], **demo)).strip()+ template = signature_to_template(predictor.signature)+ completion = demo.pop(template.fields[-1].output_variable)+ prompt = template.query(dsp.Example(demos=[], **demo)).strip()finetune_data[name_].append(dict(prompt=prompt, completion=completion))diff --git a/dspy/teleprompt/signature_opt.py b/dspy/teleprompt/signature_opt.pyindex 4c047b3da..512372f00 100644--- a/dspy/teleprompt/signature_opt.py+++ b/dspy/teleprompt/signature_opt.py@@ -48,6 +48,8 @@ Your task is to propose a new instruction that will lead a good language model tclass SignatureOptimizer(Teleprompter):def __init__(self, prompt_model=None, metric=None, breadth=10, depth=3, init_temperature=1.4, verbose=False, track_stats=False):+ if breadth <= 1:+ raise ValueError("Breadth must be greater than 1")self.metric = metricself.breadth = breadthself.depth = depth@@ -60,7 +62,9 @@ class SignatureOptimizer(Teleprompter):for p1, p2 in zip(candidate1["program"].predictors(), candidate2["program"].predictors()):if not p1.extended_signature.instructions == p2.extended_signature.instructions:return False- if not p1.extended_signature.fields[-1] == p2.extended_signature.fields[-1]:+ *_, p1_last_field = p1.extended_signature.fields.values()+ *_, p2_last_field = p2.extended_signature.fields.values()+ if not p1_last_field == p2_last_field:return Falsereturn True@@ -103,12 +107,13 @@ class SignatureOptimizer(Teleprompter):for predictor in module.predictors():basic_instruction = Nonebasic_prefix = None+ *_, last_key = predictor.extended_signature.fields.keys()if (hasattr(predictor, 'extended_signature')):basic_instruction = predictor.extended_signature.instructions- basic_prefix = predictor.extended_signature.fields[-1].name+ basic_prefix = predictor.extended_signature.fields[last_key].json_schema_extra['prefix']else:basic_instruction = predictor.extended_signature1.instructions- basic_prefix = predictor.extended_signature1.fields[-1].name+ basic_prefix = predictor.extended_signature1.fields[last_key].json_schema_extra['prefix']if self.prompt_model:with dspy.settings.context(lm=self.prompt_model):instruct = dspy.Predict(BasicGenerateInstruction, n=self.breadth-1, temperature=self.init_temperature)(basic_instruction=basic_instruction)@@ -146,13 +151,19 @@ class SignatureOptimizer(Teleprompter):# Set this new module with our instruction / prefixif (hasattr(p_new, 'extended_signature')):- p_new.extended_signature.instructions = instruction- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=prefix)+ *_, last_key = p_new.extended_signature.fields.keys()+ p_new.extended_signature = p_new.extended_signature \+ .with_instructions(instruction) \+ .with_updated_fields(last_key, prefix=prefix)else:- p_new.extended_signature1.instructions = instruction- p_new.extended_signature1.fields[-1] = p_new.extended_signature1.fields[-1]._replace(name=prefix)- p_new.extended_signature2.instructions = instruction- p_new.extended_signature2.fields[-1] = p_new.extended_signature2.fields[-1]._replace(name=prefix)+ *_, last_key = p_new.extended_signature1.fields.keys()+ p_new.extended_signature1 = p_new.extended_signature1 \+ .with_instructions(instruction) \+ .with_updated_fields(last_key, prefix=prefix)+ *_, last_key = p_new.extended_signature2.fields.keys()+ p_new.extended_signature2 = p_new.extended_signature2 \+ .with_instructions(instruction) \+ .with_updated_fields(last_key, prefix=prefix)# Score the instruction / prefixif self.verbose: print(f"----------------")@@ -203,13 +214,19 @@ class SignatureOptimizer(Teleprompter):# to ensure the next round of scores reflect the best possible versionbest_candidate = max(evaluated_candidates[id(p_old)].values(), key=lambda candidate: candidate['score'])if (hasattr(p_new, 'extended_signature')):- p_new.extended_signature.instructions = best_candidate["instruction"]- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=best_candidate["prefix"])+ *_, last_key = p_old.extended_signature.fields.keys()+ p_new.extended_signature = p_new.extended_signature \+ .with_instructions(best_candidate["instruction"]) \+ .with_updated_fields(last_key, prefix=best_candidate["prefix"])else:- p_new.extended_signature1.instructions = best_candidate["instruction"]- p_new.extended_signature1.fields[-1] = p_new.extended_signature1.fields[-1]._replace(name=best_candidate["prefix"])- p_new.extended_signature2.instructions = best_candidate["instruction"]- p_new.extended_signature2.fields[-1] = p_new.extended_signature2.fields[-1]._replace(name=best_candidate["prefix"])+ *_, last_key1 = p_old.extended_signature1.fields.keys()+ p_new.extended_signature1 = p_new.extended_signature \+ .with_instructions(best_candidate["instruction"]) \+ .with_updated_fields(last_key1, prefix=best_candidate["prefix"])+ *_, last_key2 = p_old.extended_signature2.fields.keys()+ p_new.extended_signature2 = p_new.extended_signature \+ .with_instructions(best_candidate["instruction"]) \+ .with_updated_fields(last_key2, prefix=best_candidate["prefix"])if self.verbose: print(f"Updating Predictor {id(p_old)} to:\ni: {best_candidate['instruction']}\np: {best_candidate['prefix']}")if self.verbose: print(f"Full predictor with update: ")for i,predictor in enumerate(module_clone.predictors()):diff --git a/dspy/teleprompt/signature_opt_bayesian.py b/dspy/teleprompt/signature_opt_bayesian.pyindex 68d7aacf0..045461fce 100644--- a/dspy/teleprompt/signature_opt_bayesian.py+++ b/dspy/teleprompt/signature_opt_bayesian.py@@ -1,5 +1,6 @@import dspimport dspy+from dspy.signatures.signature import signature_to_templatefrom dspy.teleprompt.teleprompt import Teleprompterfrom dspy.signatures import Signaturefrom dspy.evaluate.evaluate import Evaluate@@ -114,10 +115,12 @@ class BayesianSignatureOptimizer(Teleprompter):if self.verbose: print(f"Predictor {i}")if (hasattr(predictor, 'extended_signature')):if self.verbose: print(f"i: {predictor.extended_signature.instructions}")- if self.verbose: print(f"p: {predictor.extended_signature.fields[-1].name}")+ *_, last_field = predictor.extended_signature.fields.values()+ if self.verbose: print(f"p: {last_field.json_schema_extra['prefix']}")else:if self.verbose: print(f"i: {predictor.extended_signature1.instructions}")- if self.verbose: print(f"p: {predictor.extended_signature1.fields[-1].name}")+ *_, last_field = predictor.extended_signature1.fields.values()+ if self.verbose: print(f"p: {last_field.json_schema_extra['prefix']}")if self.verbose: print("\n")def _print_model_history(self, model, n=1):@@ -186,8 +189,8 @@ class BayesianSignatureOptimizer(Teleprompter):if example["augmented"]:if example_set_i not in example_set:example_set[example_set_i] = []- fields_to_use = predictor.signature.fields- input_variable_names = [field.input_variable for field in fields_to_use]+ fields_to_use = signature_to_template(predictor.signature).fields+ input_variable_names = list(predictor.signature.input_fields.keys())example_with_only_signature_fields = {key: value for key, value in example.items() if key in input_variable_names}example_string = self._create_example_string(fields_to_use, example_with_only_signature_fields)example_set[example_set_i].append(example_string)@@ -202,16 +205,28 @@ class BayesianSignatureOptimizer(Teleprompter):basic_prefix = Noneif (hasattr(predictor, 'extended_signature')):basic_instruction = predictor.extended_signature.instructions- basic_prefix = predictor.extended_signature.fields[-1].name+ *_, last_field = predictor.extended_signature.fields.values()+ basic_prefix = last_field.json_schema_extra["prefix"]else:basic_instruction = predictor.extended_signature1.instructions- basic_prefix = predictor.extended_signature1.fields[-1].name+ *_, last_field = predictor.extended_signature1.fields.values()+ basic_prefix = last_field.json_schema_extra["prefix"]with dspy.settings.context(lm=self.prompt_model):# Data & Examplesif view_data and view_examples:+ if 1 not in example_sets[id(predictor)].keys():+ raise ValueError("No examples found for the given predictor")instruct = None- for i in range(1,self.n):- new_instruct = dspy.Predict(BasicGenerateInstructionWithExamplesAndDataObservations, n=1, temperature=self.init_temperature)(basic_instruction=basic_instruction, observations=self.observations, examples=example_sets[id(predictor)][i])+ for i in range(1, self.n):+ new_instruct = dspy.Predict(+ BasicGenerateInstructionWithExamplesAndDataObservations,+ n=1,+ temperature=self.init_temperature+ )(+ basic_instruction=basic_instruction,+ observations=self.observations,+ examples=example_sets[id(predictor)][i]+ )if not instruct:instruct = new_instructelse:@@ -224,7 +239,14 @@ class BayesianSignatureOptimizer(Teleprompter):elif view_examples:instruct = Nonefor i in range(1,self.n): # Note: skip over the first example set which is empty- new_instruct = dspy.Predict(BasicGenerateInstructionWithExamples, n=1, temperature=self.init_temperature)(basic_instruction=basic_instruction, examples=example_sets[id(predictor)][i])+ new_instruct = dspy.Predict(+ BasicGenerateInstructionWithExamples,+ n=1,+ temperature=self.init_temperature+ )(+ basic_instruction=basic_instruction,+ examples=example_sets[id(predictor)][i]+ )if not instruct:instruct = new_instructelse:@@ -314,8 +336,10 @@ class BayesianSignatureOptimizer(Teleprompter):selected_prefix = selected_candidate.proposed_prefix_for_output_field.strip('"').strip()# Use this candidates in our program- p_new.extended_signature.instructions = selected_instruction- p_new.extended_signature.fields[-1] = p_new.extended_signature.fields[-1]._replace(name=selected_prefix)+ *_, last_field = p_new.extended_signature.fields.keys()+ p_new.extended_signature = p_new.extended_signature \+ .with_instructions(selected_instruction) \+ .with_updated_fields(last_field, prefix=selected_prefix)# Get the selected demosselected_demos = p_demo_candidates[demos_idx]@@ -353,8 +377,9 @@ class BayesianSignatureOptimizer(Teleprompter):trial_num += 1raise optuna.TrialPruned()- if self.verbose: print(f"Fully evaled score: {curr_weighted_avg_score}")- self._print_model_history(self.task_model, n=1)+ if self.verbose:+ print(f"Fully evaled score: {curr_weighted_avg_score}")+ self._print_model_history(self.task_model, n=1)score = curr_weighted_avg_scoretrial_logs[trial_num]["score"] = curr_weighted_avg_scorediff --git a/dspy/utils/__init__.py b/dspy/utils/__init__.pynew file mode 100644index 000000000..c6f239df0--- /dev/null+++ b/dspy/utils/__init__.py@@ -0,0 +1 @@+from .dummies import *\ No newline at end of filediff --git a/dspy/utils/dummies.py b/dspy/utils/dummies.pynew file mode 100644index 000000000..a0c997145--- /dev/null+++ b/dspy/utils/dummies.py@@ -0,0 +1,144 @@+import random+from dsp.modules import LM+from typing import List, Union, Dict+import numpy as np+from dsp.utils.utils import dotdict+import re+++class DummyLM(LM):+ """Dummy language model for unit testing purposes."""++ def __init__(self, answers: Union[List[str], Dict[str,str]], follow_examples: bool = False):+ """+ Initializes the dummy language model.+ Parameters:+ - answers: A list of strings or a dictionary with string keys and values.+ - follow_examples: If True, and the prompt contains an example exactly equal to the prompt,+ the dummy model will return the next string in the list for each request.+ If a list is provided, the dummy model will return the next string in the list for each request.+ If a dictionary is provided, the dummy model will return the value corresponding to the key that matches the prompt.+ """+ super().__init__("dummy-model")+ self.provider = "dummy"+ self.answers = answers+ self.follow_examples = follow_examples++ def basic_request(self, prompt, n=1, **kwargs):+ """Generates a dummy response based on the prompt."""+ dummy_response = {"choices": []}+ for _ in range(n):+ answer = None++ if self.follow_examples:+ prefix = prompt.split("\n")[-1]+ _instructions, _format, *examples, _output = prompt.split("\n---\n")+ examples_str = "\n".join(examples)+ possible_answers = re.findall(prefix + r"\s*(.*)", examples_str)+ if possible_answers:+ # We take the last answer, as the first one is just from+ # the "Follow the following format" section.+ answer = possible_answers[-1]+ print(f"DummyLM got found previous example for {prefix} with value {answer=}")+ else:+ print(f"DummyLM couldn't find previous example for {prefix=}")++ if answer is None:+ if isinstance(self.answers, dict):+ answer = next((v for k, v in self.answers.items() if k in prompt), None)+ else:+ if len(self.answers) > 0:+ answer = self.answers[0]+ self.answers = self.answers[1:]++ if answer is None:+ answer = "No more responses"++ # Mimic the structure of a real language model response.+ dummy_response["choices"].append({+ "text": answer,+ "finish_reason": "simulated completion",+ })++ RED, GREEN, RESET = '\033[91m', '\033[92m', '\033[0m'+ print("=== DummyLM ===")+ print(prompt, end="")+ print(f"{RED}{answer}{RESET}")+ print("===")++ # Simulate processing and storing the request and response.+ history_entry = {+ "prompt": prompt,+ "response": dummy_response,+ "kwargs": kwargs,+ "raw_kwargs": kwargs,+ }+ self.history.append(history_entry)++ return dummy_response++ def __call__(self, prompt, only_completed=True, return_sorted=False, **kwargs):+ """Retrieves dummy completions."""+ response = self.basic_request(prompt, **kwargs)+ choices = response["choices"]++ # Filter choices and return text completions.+ completions = [choice["text"] for choice in choices]++ return completions++ def get_convo(self, index):+ """Get the prompt + anwer from the ith message"""+ return self.history[index]['prompt'] \+ + " " \+ + self.history[index]['response']['choices'][0]['text']+++def dummy_rm(passages=()):+ if not passages:+ def inner(query:str, *, k:int, **kwargs):+ assert False, "No passages defined"+ return inner+ max_length = max(map(len, passages)) + 100+ vectorizer = DummyVectorizer(max_length)+ passage_vecs = vectorizer(passages)+ def inner(query:str, *, k:int, **kwargs):+ assert k <= len(passages)+ query_vec = vectorizer([query])[0]+ scores = passage_vecs @ query_vec+ largest_idx = (-scores).argsort()[:k]+ #return dspy.Prediction(passages=[passages[i] for i in largest_idx])+ return [dotdict(dict(long_text=passages[i])) for i in largest_idx]+ return inner+++class DummyVectorizer:+ """Simple vectorizer based on n-grams"""+ def __init__(self, max_length=100, n_gram=2):+ self.max_length = max_length+ self.n_gram = n_gram+ self.P = 10**9 + 7 # A large prime number+ random.seed(123)+ self.coeffs = [random.randrange(1, self.P) for _ in range(n_gram)]++ def _hash(self, gram):+ """Hashes a string using a polynomial hash function"""+ h = 1+ for coeff, c in zip(self.coeffs, gram):+ h = h * coeff + ord(c)+ h %= self.P+ return h % self.max_length++ def __call__(self, texts: List[str]) -> np.ndarray:+ vecs = []+ for text in texts:+ grams = [text[i:i+self.n_gram] for i in range(len(text) - self.n_gram + 1)]+ vec = [0] * self.max_length+ for gram in grams:+ vec[self._hash(gram)] += 1+ vecs.append(vec)++ vecs = np.array(vecs, dtype=np.float32)+ vecs -= np.mean(vecs, axis=1, keepdims=True)+ vecs /= np.linalg.norm(vecs, axis=1, keepdims=True) + 1e-10 # Added epsilon to avoid division by zero+ return vecsdiff --git a/examples/longformqa/DSPy_LongFormQA_Cache b/examples/longformqa/DSPy_LongFormQA_Cachenew file mode 160000index 000000000..595ce1f7d--- /dev/null+++ b/examples/longformqa/DSPy_LongFormQA_Cache@@ -0,0 +1 @@+Subproject commit 595ce1f7dfd71dd925795d5fa07f36c9f13a4c29diff --git a/examples/longformqa/longformqa_assertions.ipynb b/examples/longformqa/longformqa_assertions.ipynbindex d3059b1cb..40df53a24 100644--- a/examples/longformqa/longformqa_assertions.ipynb+++ b/examples/longformqa/longformqa_assertions.ipynb@@ -6,9 +6,7 @@"source": ["<img src=\"../../docs/images/DSPy8.png\" alt=\"DSPy7 Image\" height=\"150\"/>\n","\n",- "## **DSPy Assertions**: Asserting Computational Constraints on Foundation \n",- "\n",- "### **LongFormQA**: Generating long-form length responses to answer questions"+ "## **DSPy Assertions**: Asserting Computational Constraints on Foundation Models"]},{@@ -87,7 +85,6 @@"if not \"dspy-ai\" in {pkg.key for pkg in pkg_resources.working_set}:\n"," !pip install -U pip\n"," !pip install dspy-ai\n",- " !pip install openai~=0.28.1\n"," !pip install -e $repo_path\n","\n","import dspy\n",@@ -555,7 +552,7 @@"\n","We can also leverage **DSPy**'s advanced compiling features to enhance our program's performance. \n","\n",- "For this, we utilize the `BootstrapFewShotWithRandomSearch` teleprompter, which automatically incorporates few-shot demonstrations and conducts a random search over a candidate set to output the best compiled program. We evaluate this over the `answer_correctness` metric as our ultimate goal is indeed to generate correct answers to the `HotPotQA` questions from the paragraphs, aiming to optimize both intrinsic and extrinsic metrics as a result. \n",+ "For this, we utilize the `BootstrapFewShotWithRandomSearch` teleprompter, which automatically incorporates few-shot demonstrations and conducts a random search over a candidate set to output the best compiled program. We evaluate this over the `answer_correctness` metric as our ultimate goal is indeed to generate correct answers to the `HotPotQA` questions from the paragraphs, aiming to optimize both instrinsic and extrinsic metrics as a result. \n","\n","Let's evaluate this on the LongFormQA program first:"]@@ -594,24 +591,24 @@]},{- "cell_type": "markdown",- "metadata": {},- "source": [- "**Note** This pipeline on the other hand sets both the teacher and student with `LongFormQAWithAssertions()` to ensure the teacher correctly instructs the student with the right bootstrapped examples and the student has the chance to self-correct with **Assertions** for any examples that are still deemed incorrect."- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "longformqa = LongFormQA()\n",- "teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",- "cited_longformqa_student_teacher = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:100])\n",- "evaluate(cited_longformqa_student_teacher)"- ]- }+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "**Note** This pipeline on the other hand sets both the teacher and student with `LongFormQAWithAssertions()` to ensure the teacher correctly instructs the student with the right bootstrapped examples and the student has the chance to self-correct with **Assertions** for any examples that are still deemed incorrect."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "longformqa = LongFormQA()\n",+ "teleprompter = BootstrapFewShotWithRandomSearch(metric = answer_correctness, max_bootstrapped_demos=2, num_candidate_programs=6)\n",+ "cited_longformqa_student_teacher = teleprompter.compile(student=assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), teacher = assert_transform_module(LongFormQAWithAssertions().map_named_predictors(Retry), backtrack_handler), trainset=trainset, valset=devset[:100])\n",+ "evaluate(cited_longformqa_student_teacher)"+ ]+ }],"metadata": {"kernelspec": {diff --git a/pyproject.toml b/pyproject.tomlindex 8878b279a..6db47e0ad 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"[project]name = "dspy-ai"-version = "2.1.10"+version = "2.1.9"description = "DSPy"readme = "README.md"authors = [{name = "Omar Khattab", email = "[email protected]"}]@@ -26,17 +26,16 @@ dependencies = ["regex~=2023.10.3","ujson~=5.8.0","tqdm~=4.66.1",- "datasets~=2.14.6,<3.0.0",+ "datasets~=2.14.6","requests~=2.31.0","optuna~=3.4.0",][project.optional-dependencies]-chromadb = ["chromadb~=0.4.14"]+pinecone = ["pinecone-client~=2.2.4"]qdrant = ["qdrant-client~=1.6.2", "fastembed~=0.1.0"]+chromadb = ["chromadb~=0.4.14"]marqo = ["marqo"]-pinecone = ["pinecone-client~=2.2.4"]-weaviate = ["weaviate-client~=3.26.1"]docs = ["sphinx>=4.3.0","furo>=2023.3.27",@@ -81,12 +80,11 @@ tqdm = "^4.66.1"datasets = "^2.14.6"requests = "^2.31.0"optuna = "^3.4.0"-chromadb = {version = "^0.4.14", optional = true}+pinecone-client = {version = "^2.2.4", optional = true}+qdrant-client = {version = "^1.6.2", optional = true}fastembed = {version = "^0.1.0", optional = true}+chromadb = {version = "^0.4.14", optional = true}marqo = {version = "*", optional = true}-qdrant-client = {version = "^1.6.2", optional = true}-pinecone-client = {version = "^2.2.4", optional = true}-weaviate-client = {version = "^3.26.1", optional=true}sphinx = {version = ">=4.3.0", optional = true}furo = {version = ">=2023.3.27", optional = true}docutils = {version = "<0.17", optional = true}@@ -101,11 +99,10 @@ sphinx-reredirects = {version = "^0.1.2", optional = true}sphinx-automodapi = {version = "0.16.0", optional = true}[tool.poetry.extras]-chromadb = ["chromadb"]+pinecone = ["pinecone-client"]qdrant = ["qdrant-client", "fastembed"]+chromadb = ["chromadb"]marqo = ["marqo"]-pinecone = ["pinecone-client"]-weaviate = ["weaviate-client"]docs = ["sphinx","furo",diff --git a/requirements.txt b/requirements.txtindex 8dc43dd62..c853c2ef3 100644--- a/requirements.txt+++ b/requirements.txt@@ -7,4 +7,4 @@ ujsontqdmdatasetsrequests-optuna+optuna\ No newline at end of filediff --git a/setup.py b/setup.pyindex 6c5175e60..f50a568d5 100644--- a/setup.py+++ b/setup.py@@ -10,7 +10,7 @@ with open('requirements.txt', 'r', encoding='utf-8') as f:setup(name="dspy-ai",- version="2.1.10",+ version="2.1.9",description="DSPy",long_description=long_description,long_description_content_type='text/markdown',@@ -21,13 +21,13 @@ setup(packages=find_packages(include=['dsp.*', 'dspy.*', 'dsp', 'dspy']),python_requires='>=3.9',install_requires=requirements,- extras_require={- "chromadb": ["chromadb~=0.4.14"],- "qdrant": ["qdrant-client", "fastembed"],- "marqo": ["marqo"],- "mongodb": ["pymongo~=3.12.0"],+ extras_require={"pinecone": ["pinecone-client~=2.2.4"],+ "qdrant": ["qdrant-client~=1.6.2", "fastembed~=0.1.0"],+ "chromadb": ["chromadb~=0.4.14"],+ "marqo": ["marqo"],"weaviate": ["weaviate-client~=3.26.1"],+ "mongodb": ["pymongo~=3.12.0"],},classifiers=["Development Status :: 3 - Alpha",diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.pynew file mode 100644index 000000000..3af3ae9da--- /dev/null+++ b/tests/evaluate/test_evaluate.py@@ -0,0 +1,59 @@+import dsp, dspy+from dspy.evaluate.evaluate import Evaluate+from dspy.evaluate.metrics import answer_exact_match+from dspy.predict import Predict+from dspy.utils.dummies import DummyLM++def new_example(question, answer):+ """Helper function to create a new example."""+ return dspy.Example(+ question=question,+ answer=answer,+ ).with_inputs("question")++def test_evaluate_initialization():+ devset = [new_example("What is 1+1?", "2")]+ ev = Evaluate(+ devset=devset,+ metric=answer_exact_match,+ display_progress=False,+ )+ assert ev.devset == devset+ assert ev.metric == answer_exact_match+ assert ev.num_threads == len(devset)+ assert ev.display_progress == False++def test_evaluate_call():+ dspy.settings.configure(lm=DummyLM({"What is 1+1?": "2", "What is 2+2?": "4"}))+ devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]+ program = Predict("question -> answer")+ assert program(question="What is 1+1?").answer == "2"+ ev = Evaluate(+ devset=devset,+ metric=answer_exact_match,+ display_progress=False,+ )+ score = ev(program)+ assert score == 100.0++def test_evaluate_call_bad():+ dspy.settings.configure(lm=DummyLM({"What is 1+1?": "0", "What is 2+2?": "0"}))+ devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]+ program = Predict("question -> answer")+ ev = Evaluate(+ devset=devset,+ metric=answer_exact_match,+ display_progress=False,+ )+ score = ev(program)+ assert score == 0.0++def test_evaluate_display_table():+ devset = [new_example("What is 1+1?", "2")]+ ev = Evaluate(+ devset=devset,+ metric=answer_exact_match,+ display_table=True,+ )+ assert ev.display_table == True+diff --git a/tests/evaluate/test_metrics.py b/tests/evaluate/test_metrics.pynew file mode 100644index 000000000..f04148251--- /dev/null+++ b/tests/evaluate/test_metrics.py@@ -0,0 +1,32 @@+# FILEPATH: /Users/ahle/repos/dspy/tests/evaluate/test_metrics.py++import dsp, dspy+from dspy.evaluate.metrics import answer_exact_match+from dspy.predict import Predict++def test_answer_exact_match_string():+ example = dspy.Example(+ question="What is 1+1?",+ answer="2",+ ).with_inputs("question")+ pred = Predict("question -> answer")+ pred.answer = "2"+ assert answer_exact_match(example, pred)++def test_answer_exact_match_list():+ example = dspy.Example(+ question="What is 1+1?",+ answer=["2", "two"],+ ).with_inputs("question")+ pred = Predict("question -> answer")+ pred.answer = "2"+ assert answer_exact_match(example, pred)++def test_answer_exact_match_no_match():+ example = dspy.Example(+ question="What is 1+1?",+ answer="2",+ ).with_inputs("question")+ pred = Predict("question -> answer")+ pred.answer = "3"+ assert not answer_exact_match(example, pred)\ No newline at end of filediff --git a/tests/examples/test_baleen.py b/tests/examples/test_baleen.pynew file mode 100644index 000000000..ab1445844--- /dev/null+++ b/tests/examples/test_baleen.py@@ -0,0 +1,136 @@+import pytest+from dsp.utils import deduplicate+import dspy.evaluate+import dspy+from dspy.datasets import HotPotQA+from dspy.evaluate.evaluate import Evaluate+from dspy.teleprompt.bootstrap import BootstrapFewShot+++class GenerateAnswer(dspy.Signature):+ """Answer questions with short factoid answers."""++ context = dspy.InputField(desc="may contain relevant facts")+ question = dspy.InputField()+ answer = dspy.OutputField(desc="often between 1 and 5 words")+++class GenerateSearchQuery(dspy.Signature):+ """Write a simple search query that will help answer a complex question."""++ context = dspy.InputField(desc="may contain relevant facts")+ question = dspy.InputField()+ query = dspy.OutputField()+++class SimplifiedBaleen(dspy.Module):+ def __init__(self, passages_per_hop=3, max_hops=2):+ super().__init__()++ self.generate_query = [+ dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)+ ]+ self.retrieve = dspy.Retrieve(k=passages_per_hop)+ self.generate_answer = dspy.ChainOfThought(GenerateAnswer)+ self.max_hops = max_hops++ def forward(self, question):+ context = []++ for hop in range(self.max_hops):+ query = self.generate_query[hop](context=context, question=question).query+ passages = self.retrieve(query).passages+ context = deduplicate(context + passages)++ pred = self.generate_answer(context=context, question=question)+ return dspy.Prediction(context=context, answer=pred.answer)+++def load_hotpotqa():+ # Load the dataset.+ dataset = HotPotQA(+ train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0+ )+ # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata.+ trainset = [x.with_inputs("question") for x in dataset.train]+ devset = [x.with_inputs("question") for x in dataset.dev]+ return trainset, devset+++# @pytest.mark.slow_test+# TODO: Find a way to make this test run without openai+def _test_baleen():+ lm = dspy.OpenAI(model="gpt-3.5-turbo")+ rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")+ dspy.settings.configure(lm=lm, rm=rm)++ # Ask any question you like to this simple RAG program.+ my_question = "How many storeys are in the castle that David Gregory inherited?"++ # Get the prediction. This contains `pred.context` and `pred.answer`.+ uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program+ pred = uncompiled_baleen(my_question)++ assert pred.answer == "five"+++def validate_context_and_answer_and_hops(example, pred, trace=None):+ if not dspy.evaluate.answer_exact_match(example, pred):+ return False+ if not dspy.evaluate.answer_passage_match(example, pred):+ return False++ hops = [example.question] + [+ outputs.query for *_, outputs in trace if "query" in outputs+ ]++ if max([len(h) for h in hops]) > 100:+ return False+ if any(+ dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)+ for idx in range(2, len(hops))+ ):+ return False++ return True+++def gold_passages_retrieved(example, pred, trace=None):+ gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))+ found_titles = set(+ map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context])+ )++ return gold_titles.issubset(found_titles)+++# @pytest.mark.slow_test+# TODO: Find a way to make this test run without the slow hotpotqa dataset+def _test_compiled_baleen():+ trainset, devset = load_hotpotqa()+ lm = dspy.OpenAI(model="gpt-3.5-turbo")+ rm = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")+ dspy.settings.configure(lm=lm, rm=rm)++ uncompiled_baleen = SimplifiedBaleen() # uncompiled (i.e., zero-shot) program++ teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)+ compiled_baleen = teleprompter.compile(+ SimplifiedBaleen(),+ teacher=SimplifiedBaleen(passages_per_hop=2),+ trainset=trainset,+ )++ evaluate_on_hotpotqa = Evaluate(+ devset=devset, num_threads=1, display_progress=True, display_table=5+ )+ uncompiled_baleen_retrieval_score = evaluate_on_hotpotqa(+ uncompiled_baleen, metric=gold_passages_retrieved, display=False+ )+ # assert uncompiled_baleen_retrieval_score / 100 == 18 / 50++ compiled_baleen_retrieval_score = evaluate_on_hotpotqa(+ compiled_baleen, metric=gold_passages_retrieved+ )+ # assert compiled_baleen_retrieval_score / 100 == 27 / 50+ assert uncompiled_baleen_retrieval_score < compiled_baleen_retrieval_score\ No newline at end of filediff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.pynew file mode 100644index 000000000..8ed180cff--- /dev/null+++ b/tests/functional/test_functional.py@@ -0,0 +1,401 @@+import datetime+import json+import textwrap+import pydantic+from pydantic import Field, BaseModel, field_validator+from typing import Annotated++import pytest++import dspy+from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor+from dspy.primitives.example import Example+from dspy.teleprompt.bootstrap import BootstrapFewShot+from dspy.utils.dummies import DummyLM+++def test_simple():+ @predictor+ def hard_question(topic: str) -> str:+ """Think of a hard factual question about a topic."""++ expected = "What is the speed of light?"+ lm = DummyLM([expected])+ dspy.settings.configure(lm=lm)++ question = hard_question(topic="Physics")+ lm.inspect_history(n=2)++ assert question == expected+++def test_simple_type():+ class Question(pydantic.BaseModel):+ value: str++ @predictor+ def hard_question(topic: str) -> Question:+ """Think of a hard factual question about a topic."""++ expected = "What is the speed of light?"+ lm = DummyLM([f'{{"value": "{expected}"}}'])+ dspy.settings.configure(lm=lm)++ question = hard_question(topic="Physics")++ assert isinstance(question, Question)+ assert question.value == expected+++def test_simple_type_input():+ class Question(pydantic.BaseModel):+ value: str++ class Answer(pydantic.BaseModel):+ value: str++ @predictor+ def answer(question: Question) -> Answer:+ pass++ question = Question(value="What is the speed of light?")+ lm = DummyLM([f'{{"value": "3e8"}}'])+ dspy.settings.configure(lm=lm)++ result = answer(question=question)++ assert result == Answer(value="3e8")+++def test_simple_class():+ class Answer(pydantic.BaseModel):+ value: float+ certainty: float+ comments: list[str] = pydantic.Field(+ description="At least two comments about the answer"+ )++ class QA(dspy.Module):+ @predictor+ def hard_question(self, topic: str) -> str:+ """Think of a hard factual question about a topic. It should be answerable with a number."""++ @cot+ def answer(self, question: Annotated[str, "Question to answer"]) -> Answer:+ pass++ def forward(self, **kwargs):+ question = self.hard_question(**kwargs)+ return (question, self.answer(question=question))++ expected = Answer(+ value=3e8,+ certainty=0.9,+ comments=["It is the speed of light", "It is a constant"],+ )++ lm = DummyLM(+ [+ "What is the speed of light?",+ "Some bad reasoning, 3e8 m/s.",+ "3e8", # Bad answer 1+ "Some good reasoning...",+ expected.model_dump_json(), # Good answer+ ]+ )+ dspy.settings.configure(lm=lm)++ qa = QA()+ question, answer = qa(topic="Physics")++ assert question == "What is the speed of light?"+ assert answer == expected+++def test_simple_oop():+ class Question(pydantic.BaseModel):+ value: str++ class MySignature(dspy.Signature):+ topic: str = dspy.InputField()+ output: Question = dspy.OutputField()++ # Run the signature+ program = TypedPredictor(MySignature)+ expected = "What is the speed of light?"+ lm = DummyLM(+ [+ Question(value=expected).model_dump_json(),+ ]+ )+ dspy.settings.configure(lm=lm)++ question = program(topic="Physics").output++ assert isinstance(question, Question)+ assert question.value == expected+++def test_equivalent_signatures():+ class ClassSignature(dspy.Signature):+ input: str = dspy.InputField()+ output: str = dspy.OutputField()++ @predictor+ def output(input: str) -> str:+ pass++ function_signature = output.predictor.signature++ simple_signature = dspy.Signature("input -> output")++ assert ClassSignature.equals(function_signature)+ assert ClassSignature.equals(simple_signature)+++def test_named_params():+ class QA(FunctionalModule):+ @predictor+ def hard_question(self, topic: str) -> str:+ """Think of a hard factual question about a topic. It should be answerable with a number."""++ @cot+ def answer(self, question: str) -> str:+ pass++ qa = QA()+ named_predictors = list(qa.named_predictors())+ assert len(named_predictors) == 2+ names, _ = zip(*qa.named_predictors())+ assert set(names) == {"hard_question.predictor", "answer.predictor"}+++def test_bootstrap_effectiveness():+ class SimpleModule(FunctionalModule):+ @predictor+ def output(self, input: str) -> str:+ pass++ def forward(self, **kwargs):+ return self.output(**kwargs)++ def simple_metric(example, prediction, trace=None):+ return example.output == prediction.output++ examples = [+ ex.with_inputs("input")+ for ex in (+ Example(input="What is the color of the sky?", output="blue"),+ Example(+ input="What does the fox say?",+ output="Ring-ding-ding-ding-dingeringeding!",+ ),+ )+ ]+ trainset = [examples[0]]+ valset = [examples[1]]++ # This test verifies if the bootstrapping process improves the student's predictions+ student = SimpleModule()+ teacher = SimpleModule()+ assert student.output.predictor.signature.equals(teacher.output.predictor.signature)++ lm = DummyLM(["blue", "Ring-ding-ding-ding-dingeringeding!"], follow_examples=True)+ dspy.settings.configure(lm=lm, trace=[])++ bootstrap = BootstrapFewShot(+ metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1+ )+ compiled_student = bootstrap.compile(student, teacher=teacher, trainset=trainset)++ lm.inspect_history(n=2)++ # Check that the compiled student has the correct demos+ assert len(compiled_student.output.predictor.demos) == 1+ assert compiled_student.output.predictor.demos[0].input == trainset[0].input+ assert compiled_student.output.predictor.demos[0].output == trainset[0].output++ # Test the compiled student's prediction.+ # We are using a DummyLM with follow_examples=True, which means that+ # even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"+ # on the second output, if it seems an example that perfectly matches the+ # prompt, it will use that instead. That is why we expect "blue" here.+ prediction = compiled_student(input=trainset[0].input)+ assert prediction == trainset[0].output++ assert lm.get_convo(-1) == textwrap.dedent(+ """\+ Given the fields `input`, produce the fields `output`.++ ---++ Follow the following format.++ Input: ${input}+ Output: ${output}. Respond with a single str value++ ---++ Input: What is the color of the sky?+ Output: blue++ ---++ Input: What is the color of the sky?+ Output: blue"""+ )+++def test_regex():+ class TravelInformation(BaseModel):+ origin: str = Field(pattern=r"^[A-Z]{3}$")+ destination: str = Field(pattern=r"^[A-Z]{3}$")+ date: datetime.date++ @predictor+ def flight_information(email: str) -> TravelInformation:+ pass++ email = textwrap.dedent(+ """\+ We're excited to welcome you aboard your upcoming flight from+ John F. Kennedy International Airport (JFK) to Los Angeles International Airport (LAX)+ on December 25, 2022. Here's everything you need to know before you take off: ...+ """+ )+ lm = DummyLM(+ [+ # Example with a bad origin code.+ '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',+ # Fixed+ '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',+ ]+ )+ dspy.settings.configure(lm=lm)++ assert flight_information(email=email) == TravelInformation(+ origin="JFK", destination="LAX", date=datetime.date(2022, 12, 25)+ )+++def test_raises():+ class TravelInformation(BaseModel):+ origin: str = Field(pattern=r"^[A-Z]{3}$")+ destination: str = Field(pattern=r"^[A-Z]{3}$")+ date: datetime.date++ @predictor+ def flight_information(email: str) -> TravelInformation:+ pass++ lm = DummyLM(+ [+ "A list of bad inputs",+ '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',+ '{"origin": "JFK", "destination": "LAX", "date": "bad date"}',+ ]+ )+ dspy.settings.configure(lm=lm)++ with pytest.raises(ValueError):+ flight_information(email="Some email")+++def test_multi_errors():+ class TravelInformation(BaseModel):+ origin: str = Field(pattern=r"^[A-Z]{3}$")+ destination: str = Field(pattern=r"^[A-Z]{3}$")+ date: datetime.date++ @predictor+ def flight_information(email: str) -> TravelInformation:+ pass++ lm = DummyLM(+ [+ # First origin is wrong, then destination, then all is good+ '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',+ '{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}',+ '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',+ ]+ )+ dspy.settings.configure(lm=lm)++ assert flight_information(email="Some email") == TravelInformation(+ origin="JFK", destination="LAX", date=datetime.date(2022, 12, 25)+ )+ assert lm.get_convo(-1) == textwrap.dedent(+ """\+ Given the fields `email`, produce the fields `flight_information`.++ ---++ Follow the following format.++ Email: ${email}++ Past Error (flight_information): An error to avoid in the future++ Past Error (flight_information, 2): An error to avoid in the future++ Flight Information: ${flight_information}. Respond with a single JSON object using the schema {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}++ ---++ Email: Some email++ Past Error (flight_information): 1 validation error for TravelInformation origin String should match pattern '^[A-Z]{3}$' [type=string_pattern_mismatch, input_value='JF0', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/string_pattern_mismatch++ Past Error (flight_information, 2): 1 validation error for TravelInformation destination String should match pattern '^[A-Z]{3}$' [type=string_pattern_mismatch, input_value='LA0', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/string_pattern_mismatch++ Flight Information: {"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}"""+ )+++def test_field_validator():+ class UserDetails(BaseModel):+ name: str+ age: int++ @field_validator("name")+ @classmethod+ def validate_name(cls, v):+ if v.upper() != v:+ raise ValueError("Name must be in uppercase.")+ return v++ @predictor+ def get_user_details() -> UserDetails:+ pass++ # Keep making the mistake (lower case name) until we run+ # out of retries.+ lm = DummyLM(+ [+ '{"name": "lower case name", "age": 25}',+ ]+ * 10+ )+ dspy.settings.configure(lm=lm)++ with pytest.raises(ValueError):+ get_user_details()++ assert lm.get_convo(-1) == textwrap.dedent(+ """\+ Given the fields , produce the fields `get_user_details`.++ ---++ Follow the following format.++ Past Error (get_user_details): An error to avoid in the future+ Past Error (get_user_details, 2): An error to avoid in the future+ Get User Details: ${get_user_details}. Respond with a single JSON object using the schema {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}++ ---++ Past Error (get_user_details): 1 validation error for UserDetails name Value error, Name must be in uppercase. [type=value_error, input_value='lower case name', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/value_error+ Past Error (get_user_details, 2): 1 validation error for UserDetails name Value error, Name must be in uppercase. [type=value_error, input_value='lower case name', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/value_error+ Get User Details: {"name": "lower case name", "age": 25}"""+ )diff --git a/tests/predict/test_aggregation.py b/tests/predict/test_aggregation.pynew file mode 100644index 000000000..2c5f705fe--- /dev/null+++ b/tests/predict/test_aggregation.py@@ -0,0 +1,47 @@+from dspy.predict.aggregation import majority+from dspy.primitives.prediction import Prediction, Completions+from dsp.utils import normalize_text+++def test_majority_with_prediction():+ prediction = Prediction.from_completions(+ [{"answer": "2"}, {"answer": "2"}, {"answer": "3"}]+ )+ result = majority(prediction)+ assert result.completions[0]["answer"] == "2"+++def test_majority_with_completions():+ completions = Completions([{"answer": "2"}, {"answer": "2"}, {"answer": "3"}])+ result = majority(completions)+ assert result.completions[0]["answer"] == "2"+++def test_majority_with_list():+ completions = [{"answer": "2"}, {"answer": "2"}, {"answer": "3"}]+ result = majority(completions)+ assert result.completions[0]["answer"] == "2"+++def test_majority_with_normalize():+ completions = [{"answer": "2"}, {"answer": " 2"}, {"answer": "3"}]+ result = majority(completions, normalize=normalize_text)+ assert result.completions[0]["answer"] == "2"+++def test_majority_with_field():+ completions = [+ {"answer": "2", "other": "1"},+ {"answer": "2", "other": "1"},+ {"answer": "3", "other": "2"},+ ]+ result = majority(completions, field="other")+ assert result.completions[0]["other"] == "1"+++def test_majority_with_no_majority():+ completions = [{"answer": "2"}, {"answer": "3"}, {"answer": "4"}]+ result = majority(completions)+ assert (+ result.completions[0]["answer"] == "2"+ ) # The first completion is returned in case of a tiediff --git a/tests/predict/test_chain_of_thought.py b/tests/predict/test_chain_of_thought.pynew file mode 100644index 000000000..c1d08e729--- /dev/null+++ b/tests/predict/test_chain_of_thought.py@@ -0,0 +1,35 @@+import textwrap+import dspy+from dspy import ChainOfThought+from dspy.utils import DummyLM+++def test_initialization_with_string_signature():+ lm = DummyLM(["find the number after 1", "2"])+ dspy.settings.configure(lm=lm)+ predict = ChainOfThought("question -> answer")+ assert list(predict.extended_signature.output_fields.keys()) == [+ "rationale",+ "answer",+ ]+ assert predict(question="What is 1+1?").answer == "2"++ print(lm.get_convo(-1))+ assert lm.get_convo(-1) == textwrap.dedent(+ """\+ Given the fields `question`, produce the fields `answer`.++ ---++ Follow the following format.++ Question: ${question}+ Reasoning: Let's think step by step in order to ${produce the answer}. We ...+ Answer: ${answer}++ ---++ Question: What is 1+1?+ Reasoning: Let's think step by step in order to find the number after 1+ Answer: 2"""+ )diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.pynew file mode 100644index 000000000..b5e62425d--- /dev/null+++ b/tests/predict/test_chain_of_thought_with_hint.py@@ -0,0 +1,42 @@+import dspy+from dspy import ChainOfThoughtWithHint+from dspy.utils import DummyLM+++def test_cot_with_no_hint():+ lm = DummyLM(["find the number after 1", "2"])+ dspy.settings.configure(lm=lm)+ predict = ChainOfThoughtWithHint("question -> answer")+ assert list(predict.extended_signature2.output_fields.keys()) == [+ "rationale",+ "hint",+ "answer",+ ]+ assert predict(question="What is 1+1?").answer == "2"++ final_convo = lm.get_convo(-1)+ assert final_convo.endswith(+ "Question: What is 1+1?\n"+ "Reasoning: Let's think step by step in order to find the number after 1\n"+ "Answer: 2"+ )+++def test_cot_with_hint():+ lm = DummyLM(["find the number after 1", "2"])+ dspy.settings.configure(lm=lm)+ predict = ChainOfThoughtWithHint("question -> answer")+ assert list(predict.extended_signature2.output_fields.keys()) == [+ "rationale",+ "hint",+ "answer",+ ]+ assert predict(question="What is 1+1?", hint="think small").answer == "2"++ final_convo = lm.get_convo(-1)+ assert final_convo.endswith(+ "Question: What is 1+1?\n\n"+ "Reasoning: Let's think step by step in order to find the number after 1\n\n"+ "Hint: think small\n\n"+ "Answer: 2"+ )diff --git a/tests/predict/test_knn.py b/tests/predict/test_knn.pynew file mode 100644index 000000000..62cf96682--- /dev/null+++ b/tests/predict/test_knn.py@@ -0,0 +1,55 @@+import pytest+import numpy as np+import dsp, dspy+from dspy.utils import DummyVectorizer+from dspy.predict import KNN+++def mock_example(question: str, answer: str) -> dsp.Example:+ """Creates a mock DSP example with specified question and answer."""+ return dspy.Example(question=question, answer=answer).with_inputs("question")+++def setup_knn():+ """Sets up a KNN instance with a mocked vectorizer for testing."""+ dsp.SentenceTransformersVectorizer = DummyVectorizer+ trainset = [+ mock_example("What is the capital of France?", "Paris"),+ mock_example("What is the largest ocean?", "Pacific"),+ mock_example("What is 2+2?", "4"),+ ]+ knn = KNN(k=2, trainset=trainset)+ return knn+++def test_knn_initialization(setup_knn):+ """Tests the KNN initialization and checks if the trainset vectors are correctly created."""+ knn = setup_knn+ assert knn.k == 2, "Incorrect k value"+ assert len(knn.trainset_vectors) == 3, "Incorrect size of trainset vectors"+ assert isinstance(+ knn.trainset_vectors, np.ndarray+ ), "Trainset vectors should be a NumPy array"+++def test_knn_query(setup_knn):+ """Tests the KNN query functionality for retrieving the nearest neighbors."""+ knn = setup_knn+ query = {"question": "What is 3+3?"} # A query close to "What is 2+2?"+ nearest_samples = knn(**query)+ assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"+ assert nearest_samples[0].answer == "4", "Incorrect nearest sample returned"+++def test_knn_query_specificity(setup_knn):+ """Tests the KNN query functionality for specificity of returned examples."""+ knn = setup_knn+ query = {+ "question": "What is the capital of Germany?"+ } # A query close to "What is the capital of France?"+ nearest_samples = knn(**query)+ assert len(nearest_samples) == 2, "Incorrect number of nearest samples returned"+ assert "Paris" in [+ sample.answer for sample in nearest_samples+ ], "Expected Paris to be a nearest sample answer"diff --git a/tests/predict/test_multi_chain_comparison.py b/tests/predict/test_multi_chain_comparison.pynew file mode 100644index 000000000..8c936a2d8--- /dev/null+++ b/tests/predict/test_multi_chain_comparison.py@@ -0,0 +1,38 @@+import dspy+from dspy.utils.dummies import DummyLM+++def test_basic_example():+ class BasicQA(dspy.Signature):+ """Answer questions with short factoid answers."""++ question = dspy.InputField()+ answer = dspy.OutputField(desc="often between 1 and 5 words")++ # Example completions generated by a model for reference+ completions = [+ dspy.Prediction(+ rationale="I recall that during clear days, the sky often appears this color.",+ answer="blue",+ ),+ dspy.Prediction(+ rationale="Based on common knowledge, I believe the sky is typically seen as this color.",+ answer="green",+ ),+ dspy.Prediction(+ rationale="From images and depictions in media, the sky is frequently represented with this hue.",+ answer="blue",+ ),+ ]++ # Pass signature to MultiChainComparison module+ compare_answers = dspy.MultiChainComparison(BasicQA)++ # Call the MultiChainComparison on the completions+ question = "What is the color of the sky?"+ lm = DummyLM(["my rationale", "blue"])+ dspy.settings.configure(lm=lm)+ final_pred = compare_answers(completions, question=question)++ assert final_pred.rationale == "my rationale"+ assert final_pred.answer == "blue"diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.pynew file mode 100644index 000000000..e44b3a135--- /dev/null+++ b/tests/predict/test_predict.py@@ -0,0 +1,91 @@+import dspy+from dspy import Predict, Signature+from dspy.utils.dummies import DummyLM+++def test_initialization_with_string_signature():+ signature_string = "input1, input2 -> output"+ predict = Predict(signature_string)+ expected_instruction = (+ "Given the fields `input1`, `input2`, produce the fields `output`."+ )+ assert predict.signature.instructions == expected_instruction+ assert predict.signature.instructions == Signature(signature_string).instructions+++def test_reset_method():+ predict_instance = Predict("input -> output")+ predict_instance.lm = "modified"+ predict_instance.traces = ["trace"]+ predict_instance.train = ["train"]+ predict_instance.demos = ["demo"]+ predict_instance.reset()+ assert predict_instance.lm is None+ assert predict_instance.traces == []+ assert predict_instance.train == []+ assert predict_instance.demos == []+++def test_dump_and_load_state():+ predict_instance = Predict("input -> output")+ predict_instance.lm = "lm_state"+ dumped_state = predict_instance.dump_state()+ new_instance = Predict("input -> output")+ new_instance.load_state(dumped_state)+ assert new_instance.lm == "lm_state"+++def test_call_method():+ predict_instance = Predict("input -> output")+ lm = DummyLM(["test output"])+ dspy.settings.configure(lm=lm)+ result = predict_instance(input="test input")+ assert result.output == "test output"+ assert lm.get_convo(-1) == (+ "Given the fields `input`, produce the fields `output`.\n"+ "\n---\n\n"+ "Follow the following format.\n\n"+ "Input: ${input}\n"+ "Output: ${output}\n"+ "\n---\n\n"+ "Input: test input\n"+ "Output: test output"+ )+++def test_dump_load_state():+ predict_instance = Predict(Signature("input -> output", "original instructions"))+ dumped_state = predict_instance.dump_state()+ new_instance = Predict(Signature("input -> output", "new instructions"))+ new_instance.load_state(dumped_state)+ assert new_instance.signature.instructions == "original instructions"+++def test_forward_method():+ program = Predict("question -> answer")+ dspy.settings.configure(lm=DummyLM([]))+ result = program(question="What is 1+1?").answer+ assert result == "No more responses"+++def test_forward_method2():+ program = Predict("question -> answer1, answer2")+ dspy.settings.configure(lm=DummyLM(["my first answer", "my second answer"]))+ result = program(question="What is 1+1?")+ assert result.answer1 == "my first answer"+ assert result.answer2 == "my second answer"+++def test_config_management():+ predict_instance = Predict("input -> output")+ predict_instance.update_config(new_key="value")+ config = predict_instance.get_config()+ assert "new_key" in config and config["new_key"] == "value"+++def test_multi_output():+ program = Predict("question -> answer", n=2)+ dspy.settings.configure(lm=DummyLM(["my first answer", "my second answer"]))+ results = program(question="What is 1+1?")+ assert results.completions.answer[0] == "my first answer"+ assert results.completions.answer[1] == "my second answer"diff --git a/tests/predict/test_program_of_thought.py b/tests/predict/test_program_of_thought.pynew file mode 100644index 000000000..2aa153a1d--- /dev/null+++ b/tests/predict/test_program_of_thought.py@@ -0,0 +1,121 @@+from dspy import Signature, ProgramOfThought+import dspy+from dspy.utils import DummyLM+import textwrap++class BasicQA(Signature):+ question = dspy.InputField()+ answer = dspy.OutputField(desc="often between 1 and 5 words")++def test_pot_code_generation():+ pot = ProgramOfThought(BasicQA)+ lm = DummyLM([+ "Reason_A",+ "```python\nresult = 1+1\n```",+ "Reason_B",+ "2",+ ])+ dspy.settings.configure(lm=lm)+ res = pot(question="What is 1+1?")+ assert res.answer == "2"+ assert lm.get_convo(index=-1) == textwrap.dedent("""\+ Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`.++ ---++ Follow the following format.++ Question: ${question}++ Code: python code that answers the question++ Code Output: output of previously-generated python code++ Reasoning: Let's think step by step in order to ${produce the answer}. We ...++ Answer: often between 1 and 5 words++ ---++ Question: What is 1+1?++ Code: result = 1+1++ Code Output: 2++ Reasoning: Let's think step by step in order to Reason_B++ Answer: 2""")++def test_pot_code_generation_with_error():+ pot = ProgramOfThought(BasicQA)+ lm = DummyLM([+ "Reason_A",+ "```python\nresult = 1+0/0\n```",+ "Reason_B", # Error: division by zero+ "```python\nresult = 1+1\n```",+ "Reason_C",+ "2",+ ])+ dspy.settings.configure(lm=lm)+ res = pot(question="What is 1+1?")+ assert res.answer == "2"++ # The first code example failed+ assert lm.get_convo(index=2) == textwrap.dedent("""\+ You are given `question`, `previous_code`, `error` due to an error in previous code.+ Your task is to correct the error and provide the new `generated_code`.++ ---++ Follow the following format.++ Question: ${question}++ Previous Code: previously-generated python code that errored++ Error: error message from previously-generated python code++ Reasoning: Let's think step by step in order to ${produce the generated_code}. We ...++ Code: python code that answers the question++ ---++ Question: What is 1+1?++ Previous Code: result = 1+0/0++ Error: division by zero++ Reasoning: Let's think step by step in order to Reason_B""")++ # The second code example succeeded+ assert lm.get_convo(-1) == textwrap.dedent("""\+ Given the final code `question`, `final_generated_code`, `code_output`, provide the final `answer`.++ ---++ Follow the following format.++ Question: ${question}++ Code: python code that answers the question++ Code Output: output of previously-generated python code++ Reasoning: Let's think step by step in order to ${produce the answer}. We ...++ Answer: often between 1 and 5 words++ ---++ Question: What is 1+1?++ Code: result = 1+1++ Code Output: 2++ Reasoning: Let's think step by step in order to Reason_C++ Answer: 2""")diff --git a/tests/predict/test_react.py b/tests/predict/test_react.pynew file mode 100644index 000000000..f28e905e7--- /dev/null+++ b/tests/predict/test_react.py@@ -0,0 +1,86 @@+import dspy+from dspy.utils.dummies import dummy_rm+++def test_example_no_tools():+ # Createa a simple dataset which the model will use with the Retrieve tool.+ lm = dspy.utils.DummyLM(+ [+ "Initial thoughts", # Thought_1+ "Finish[blue]", # Action_1+ ]+ )+ dspy.settings.configure(lm=lm, rm=dummy_rm())++ program = dspy.ReAct("question -> answer")++ # Check default tools+ assert isinstance(program.tools["Finish"], dspy.Example)++ # Call the ReAct module on a particular input+ question = "What is the color of the sky?"+ result = program(question=question)+ assert result.answer == "blue"++ # For debugging+ print("---")+ for row in lm.history:+ print(row["prompt"])+ print("Response:", row["response"]["choices"][0]["text"])+ print("---")++ assert lm.get_convo(-1).endswith(+ "Question: What is the color of the sky?\n"+ "Thought 1: Initial thoughts\n"+ "Action 1: Finish[blue]"+ )+++def test_example_search():+ # Createa a simple dataset which the model will use with the Retrieve tool.+ lm = dspy.utils.DummyLM(+ [+ "Initial thoughts", # Thought_1+ "Search[the color of the sky]", # Thought_1+ "More thoughts", # Thought_2+ "Finish[blue]", # Action_2+ ]+ )+ rm = dummy_rm(+ [+ "We all know the color of the sky is blue.",+ "Somethng about the sky colors",+ "This sentence is completely irellevant to answer the question.",+ "Let's add some more sentences to act as summy passages.",+ "Let's add some more sentences to act as summy passages.",+ "Let's add some more sentences to act as summy passages.",+ ]+ )+ dspy.settings.configure(lm=lm, rm=rm)++ program = dspy.ReAct("question -> answer")++ # Check default tools+ assert len(program.tools) == 2+ assert isinstance(program.tools["Search"], dspy.Retrieve)+ assert isinstance(program.tools["Finish"], dspy.Example)++ # Call the ReAct module on a particular input+ question = "What is the color of the sky?"+ result = program(question=question)+ assert result.answer == "blue"++ # For debugging+ print(lm.get_convo(-1))++ assert lm.get_convo(-1).endswith(+ "Question: What is the color of the sky?\n\n"+ "Thought 1: Initial thoughts\n\n"+ "Action 1: Search[the color of the sky]\n\n"+ "Observation 1:\n"+ "[1] «We all know the color of the sky is blue.»\n"+ "[2] «Somethng about the sky colors»\n"+ "[3] «This sentence is completely irellevant to answer the question.»\n\n"+ "Thought 2: More thoughts\n\n"+ "Action 2: Finish[blue]"+ )diff --git a/tests/predict/test_retry.py b/tests/predict/test_retry.pynew file mode 100644index 000000000..a125dde29--- /dev/null+++ b/tests/predict/test_retry.py@@ -0,0 +1,66 @@+import functools+import dspy+from dspy.utils import DummyLM+from dspy.primitives.assertions import assert_transform_module, backtrack_handler+++def test_retry_simple():+ predict = dspy.Predict("question -> answer")+ retry_module = dspy.Retry(predict)++ # Test Retry has created the correct new signature+ for field in predict.signature.output_fields:+ assert f"past_{field}" in retry_module.new_signature.input_fields+ assert "feedback" in retry_module.new_signature.input_fields++ lm = DummyLM(["blue"])+ dspy.settings.configure(lm=lm)+ result = retry_module.forward(+ question="What color is the sky?",+ past_outputs={"answer": "red"},+ feedback="Try harder",+ )+ assert result.answer == "blue"++ print(lm.get_convo(-1))+ assert lm.get_convo(-1).endswith(+ "Question: What color is the sky?\n\n"+ "Past Answer: red\n\n"+ "Instructions: Try harder\n\n"+ "Answer: blue"+ )+++def test_retry_forward_with_feedback():+ # First we make a mistake, then we fix it+ lm = DummyLM(["red", "blue"])+ dspy.settings.configure(lm=lm, trace=[])++ class SimpleModule(dspy.Module):+ def __init__(self):+ super().__init__()+ self.predictor = dspy.Predict("question -> answer")++ def forward(self, **kwargs):+ result = self.predictor(**kwargs)+ print(f"SimpleModule got {result.answer=}")+ dspy.Suggest(result.answer == "blue", "Please think harder")+ return result++ program = SimpleModule()+ program = assert_transform_module(+ program.map_named_predictors(dspy.Retry),+ functools.partial(backtrack_handler, max_backtracks=1),+ )++ result = program(question="What color is the sky?")++ assert result.answer == "blue"++ print(lm.get_convo(-1))+ assert lm.get_convo(-1).endswith(+ "Question: What color is the sky?\n\n"+ "Past Answer: red\n\n"+ "Instructions: Please think harder\n\n"+ "Answer: blue"+ )diff --git a/tests/primitives/test_example.py b/tests/primitives/test_example.pynew file mode 100644index 000000000..2f27996a2--- /dev/null+++ b/tests/primitives/test_example.py@@ -0,0 +1,108 @@+import pytest+from dspy import Example+++def test_example_initialization():+ example = Example(a=1, b=2)+ assert example.a == 1+ assert example.b == 2+++def test_example_initialization_from_base():+ base = Example(a=1, b=2)+ example = Example(base=base, c=3)+ assert example.a == 1+ assert example.b == 2+ assert example.c == 3+++def test_example_initialization_from_dict():+ base_dict = {"a": 1, "b": 2}+ example = Example(base=base_dict, c=3)+ assert example.a == 1+ assert example.b == 2+ assert example.c == 3+++def test_example_set_get_item():+ example = Example()+ example["a"] = 1+ assert example["a"] == 1+++def test_example_attribute_access():+ example = Example(a=1)+ assert example.a == 1+ example.a = 2+ assert example.a == 2+++def test_example_deletion():+ example = Example(a=1, b=2)+ del example["a"]+ with pytest.raises(AttributeError):+ _ = example.a+++def test_example_len():+ example = Example(a=1, b=2, dspy_hidden=3)+ assert len(example) == 2+++def test_example_repr_str():+ example = Example(a=1)+ assert repr(example) == "Example({'a': 1}) (input_keys=None)"+ assert str(example) == "Example({'a': 1}) (input_keys=None)"+++def test_example_eq():+ example1 = Example(a=1, b=2)+ example2 = Example(a=1, b=2)+ assert example1 == example2+++def test_example_hash():+ example1 = Example(a=1, b=2)+ example2 = Example(a=1, b=2)+ assert hash(example1) == hash(example2)+++def test_example_keys_values_items():+ example = Example(a=1, b=2, dspy_hidden=3)+ assert set(example.keys()) == {"a", "b"}+ assert 1 in example.values()+ assert ("b", 2) in example.items()+++def test_example_get():+ example = Example(a=1, b=2)+ assert example.get("a") == 1+ assert example.get("c", "default") == "default"+++def test_example_with_inputs():+ example = Example(a=1, b=2).with_inputs("a")+ assert example._input_keys == {"a"}+++def test_example_inputs_labels():+ example = Example(a=1, b=2).with_inputs("a")+ inputs = example.inputs()+ assert inputs.toDict() == {"a": 1}+ labels = example.labels()+ assert labels.toDict() == {"b": 2}+++def test_example_copy_without():+ example = Example(a=1, b=2)+ copied = example.copy(c=3)+ assert copied.a == 1+ assert copied.c == 3+ without_a = copied.without("a")+ with pytest.raises(AttributeError):+ _ = without_a.a+++def test_example_to_dict():+ example = Example(a=1, b=2)+ assert example.toDict() == {"a": 1, "b": 2}diff --git a/tests/primitives/test_program.py b/tests/primitives/test_program.pynew file mode 100644index 000000000..b1d7c8972--- /dev/null+++ b/tests/primitives/test_program.py@@ -0,0 +1,66 @@+import dspy+from dspy.primitives.program import (+ Module,+ set_attribute_by_name,+) # Adjust the import based on your file structure+from dspy.utils import DummyLM+++class HopModule(dspy.Module):+ def __init__(self):+ super().__init__()+ self.predict1 = dspy.Predict("question -> query")+ self.predict2 = dspy.Predict("query -> answer")++ def forward(self, question):+ query = self.predict1(question=question).query+ return self.predict2(query=query)+++def test_module_initialization():+ module = Module()+ assert (+ module._compiled is False+ ), "Module _compiled attribute should be False upon initialization"+++def test_named_predictors():+ module = HopModule()+ named_preds = module.named_predictors()+ assert len(named_preds) == 2, "Should identify correct number of Predict instances"+ names, preds = zip(*named_preds)+ assert (+ "predict1" in names and "predict2" in names+ ), "Named predictors should include 'predict1' and 'predict2'"+++def test_predictors():+ module = HopModule()+ preds = module.predictors()+ assert len(preds) == 2, "Should return correct number of Predict instances"+ assert all(+ isinstance(p, dspy.Predict) for p in preds+ ), "All returned items should be instances of PredictMock"+++def test_forward():+ program = HopModule()+ dspy.settings.configure(+ lm=DummyLM({"What is 1+1?": "let me check", "let me check": "2"})+ )+ result = program(question="What is 1+1?").answer+ assert result == "2"+++def test_nested_named_predictors():+ class Hop2Module(dspy.Module):+ def __init__(self):+ super().__init__()+ self.hop = HopModule()++ module = Hop2Module()+ named_preds = module.named_predictors()+ assert len(named_preds) == 2+ names, _preds = zip(*named_preds)+ assert "hop.predict1" in names+ assert "hop.predict2" in namesdiff --git a/tests/primitives/test_python_interpreter.py b/tests/primitives/test_python_interpreter.pynew file mode 100644index 000000000..14b15d557--- /dev/null+++ b/tests/primitives/test_python_interpreter.py@@ -0,0 +1,44 @@+import pytest+from dspy.primitives.python_interpreter import PythonInterpreter, TextPrompt, CodePrompt++def test_execute_simple_code():+ interpreter = PythonInterpreter(action_space={'print': print})+ code = "print('Hello, World!')"+ result = interpreter.execute(code)+ assert result is None, "Simple print statement should return None"++def test_action_space_limitation():+ def func(string):+ pass+ interpreter = PythonInterpreter(action_space={})+ code = "func('This should not execute')"+ with pytest.raises(Exception):+ interpreter.execute(code)++def test_import_whitelist():+ interpreter = PythonInterpreter(action_space={}, import_white_list=['math'])+ code = "import math\nresult = math.sqrt(4)"+ result = interpreter.execute(code)+ assert result == 2, "Should be able to import and use math.sqrt"++def test_fuzzy_variable_matching():+ interpreter = PythonInterpreter(action_space={})+ code = "result = number + 1"+ result = interpreter.execute(code, fuzz_state={'number': 4})+ assert result == 5, "Fuzzy variable matching should work"++def test_text_prompt_keyword_extraction():+ prompt = TextPrompt("Hello {name}, how are you?")+ assert 'name' in prompt.key_words, "Keyword 'name' should be extracted"++def test_text_prompt_formatting():+ prompt = TextPrompt("Hello {name}, how are you?")+ formatted = prompt.format(name="Alice")+ assert formatted == "Hello Alice, how are you?", "Should format with provided value"++def test_code_prompt_execution():+ action_space = {'len': len}+ interpreter = PythonInterpreter(action_space=action_space)+ code_prompt = CodePrompt("result = len('hello')")+ result, _ = code_prompt.execute(interpreter)+ assert result == 5, "Code execution should return the length of 'hello'"diff --git a/tests/signatures/test_signature.py b/tests/signatures/test_signature.pynew file mode 100644index 000000000..b09325854--- /dev/null+++ b/tests/signatures/test_signature.py@@ -0,0 +1,166 @@+import pytest+import pydantic+from dspy import Signature, infer_prefix, InputField, OutputField+from typing import List+++def test_field_types_and_custom_attributes():+ class TestSignature(Signature):+ """Instructions"""++ input1: str = InputField()+ input2: int = InputField()+ output1: List[str] = OutputField()+ output2 = OutputField()++ assert TestSignature.instructions == "Instructions"+ assert TestSignature.input_fields["input1"].annotation == str+ assert TestSignature.input_fields["input2"].annotation == int+ assert TestSignature.output_fields["output1"].annotation == List[str]+ assert TestSignature.output_fields["output2"].annotation == str+++def test_no_input_output():+ with pytest.raises(TypeError):++ class TestSignature(Signature):+ input1: str+++def test_no_input_output2():+ with pytest.raises(TypeError):++ class TestSignature(Signature):+ input1: str = pydantic.Field()+++def test_all_fields_have_prefix():+ class TestSignature(Signature):+ input = InputField(prefix="Modified:")+ output = OutputField()++ assert (+ TestSignature.input_fields["input"].json_schema_extra["prefix"] == "Modified:"+ )+ assert (+ TestSignature.output_fields["output"].json_schema_extra["prefix"] == "Output:"+ )+++def test_signature_parsing():+ signature = Signature("input1, input2 -> output")+ assert "input1" in signature.input_fields+ assert "input2" in signature.input_fields+ assert "output" in signature.output_fields+++def test_with_signature():+ signature1 = Signature("input1, input2 -> output")+ signature2 = signature1.with_instructions("This is a test")+ assert signature2.instructions == "This is a test"+ assert signature1 is not signature2, "The type should be immutable"+++def test_with_updated_field():+ signature1 = Signature("input1, input2 -> output")+ signature2 = signature1.with_updated_fields("input1", prefix="Modified:")+ assert signature2.input_fields["input1"].json_schema_extra["prefix"] == "Modified:"+ assert signature1.input_fields["input1"].json_schema_extra["prefix"] == "Input 1:"+ assert signature1 is not signature2, "The type should be immutable"+ for key in signature1.fields.keys():+ if key != "input1":+ assert (+ signature1.fields[key].json_schema_extra+ == signature2.fields[key].json_schema_extra+ )+ assert signature1.instructions == signature2.instructions+++def test_empty_signature():+ with pytest.raises(ValueError):+ Signature("")+++def test_instructions_signature():+ with pytest.raises(ValueError):+ Signature("")+++def test_signature_instructions():+ sig1 = Signature("input1 -> output1", instructions="This is a test")+ assert sig1.instructions == "This is a test"+++def test_signature_instructions_none():+ sig1 = Signature("a, b -> c")+ assert sig1.instructions == f"Given the fields `a`, `b`, produce the fields `c`."+++def test_signature_from_dict():+ signature = Signature(+ {"input1": InputField(), "input2": InputField(), "output": OutputField()}+ )+ for k in ["input1", "input2", "output"]:+ assert k in signature.fields+ assert signature.fields[k].annotation == str+++def test_signature_from_dict():+ signature = Signature(+ {"input1": InputField(), "input2": InputField(), "output": OutputField()}+ )+ assert "input1" in signature.input_fields+ assert "input2" in signature.input_fields+ assert "output" in signature.output_fields+++def test_signature_equality():+ sig1 = Signature("input1 -> output1")+ sig2 = Signature("input1 -> output1")+ assert sig1.equals(sig2)+++def test_signature_inequality():+ sig1 = Signature("input1 -> output1")+ sig2 = Signature("input2 -> output2")+ assert not sig1.equals(sig2)+++def test_equality_format():+ class TestSignature(Signature):+ input = InputField(format=lambda x: x)+ output = OutputField()++ assert TestSignature.equals(TestSignature)+++def test_signature_reverse():+ sig = Signature("input1 -> output1")+ assert sig.signature == "input1 -> output1"+++def test_insert_field_at_various_positions():+ class InitialSignature(Signature):+ input1: str = InputField()+ output1: int = OutputField()++ S1 = InitialSignature.prepend("new_input_start", InputField(), str)+ S2 = InitialSignature.append("new_input_end", InputField(), str)+ assert "new_input_start" == list(S1.input_fields.keys())[0]+ assert "new_input_end" == list(S2.input_fields.keys())[-1]++ S3 = InitialSignature.prepend("new_output_start", OutputField(), str)+ S4 = InitialSignature.append("new_output_end", OutputField(), str)+ assert "new_output_start" == list(S3.output_fields.keys())[0]+ assert "new_output_end" == list(S4.output_fields.keys())[-1]+++def test_infer_prefix():+ assert infer_prefix("someAttributeName42IsCool") == "Some Attribute Name 42 Is Cool"+ assert infer_prefix("version2Update") == "Version 2 Update"+ assert infer_prefix("modelT45Enhanced") == "Model T 45 Enhanced"+ assert infer_prefix("someAttributeName") == "Some Attribute Name"+ assert infer_prefix("some_attribute_name") == "Some Attribute Name"+ assert infer_prefix("URLAddress") == "URL Address"+ assert infer_prefix("isHTTPSecure") == "Is HTTP Secure"+ assert infer_prefix("isHTTPSSecure123") == "Is HTTPS Secure 123"diff --git a/tests/teleprompt/test_bootstrap.py b/tests/teleprompt/test_bootstrap.pynew file mode 100644index 000000000..4758a5aae--- /dev/null+++ b/tests/teleprompt/test_bootstrap.py@@ -0,0 +1,180 @@+import pytest+import dspy+from dspy.predict import Predict+from dspy.utils.dummies import DummyLM+from dspy import Example+from dspy.teleprompt import BootstrapFewShot+import textwrap+++# Define a simple metric function for testing+def simple_metric(example, prediction, trace=None):+ # Simplified metric for testing: true if prediction matches expected output+ return example.output == prediction.output+++examples = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ Example(+ input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!"+ ),+]+trainset = [examples[0]]+valset = [examples[1]]+++def test_bootstrap_initialization():+ # Initialize BootstrapFewShot with a dummy metric and minimal setup+ bootstrap = BootstrapFewShot(+ metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1+ )+ assert bootstrap.metric == simple_metric, "Metric not correctly initialized"+++class SimpleModule(dspy.Module):+ def __init__(self, signature):+ super().__init__()+ self.predictor = Predict(signature)++ def forward(self, **kwargs):+ return self.predictor(**kwargs)+++def test_compile_with_predict_instances():+ # Create Predict instances for student and teacher+ # Note that dspy.Predict is not itself a module, so we can't use it directly here+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output")++ lm = DummyLM(["Initial thoughts", "Finish[blue]"])+ dspy.settings.configure(lm=lm)++ # Initialize BootstrapFewShot and compile the student+ bootstrap = BootstrapFewShot(+ metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1+ )+ compiled_student = bootstrap.compile(+ student, teacher=teacher, trainset=trainset, valset=valset+ )++ assert compiled_student is not None, "Failed to compile student"+ assert (+ hasattr(compiled_student, "_compiled") and compiled_student._compiled+ ), "Student compilation flag not set"+++def test_bootstrap_effectiveness():+ # This test verifies if the bootstrapping process improves the student's predictions+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output")+ lm = DummyLM(["blue", "Ring-ding-ding-ding-dingeringeding!"], follow_examples=True)+ dspy.settings.configure(lm=lm, trace=[])++ bootstrap = BootstrapFewShot(+ metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1+ )+ compiled_student = bootstrap.compile(+ student, teacher=teacher, trainset=trainset, valset=valset+ )++ # Check that the compiled student has the correct demos+ assert len(compiled_student.predictor.demos) == 1+ assert compiled_student.predictor.demos[0].input == trainset[0].input+ assert compiled_student.predictor.demos[0].output == trainset[0].output++ # Test the compiled student's prediction.+ # We are using a DummyLM with follow_examples=True, which means that+ # even though it would normally reply with "Ring-ding-ding-ding-dingeringeding!"+ # on the second output, if it seems an example that perfectly matches the+ # prompt, it will use that instead. That is why we expect "blue" here.+ prediction = compiled_student(input=trainset[0].input)+ assert prediction.output == trainset[0].output++ # For debugging+ print("Convo")+ print(lm.get_convo(-1))++ assert lm.get_convo(-1) == textwrap.dedent(+ """\+ Given the fields `input`, produce the fields `output`.++ ---++ Follow the following format.++ Input: ${input}+ Output: ${output}++ ---++ Input: What is the color of the sky?+ Output: blue++ ---++ Input: What is the color of the sky?+ Output: blue"""+ )+++def test_error_handling_during_bootstrap():+ """+ Test to verify error handling during the bootstrapping process+ """++ class BuggyModule(dspy.Module):+ def __init__(self, signature):+ super().__init__()+ self.predictor = Predict(signature)++ def forward(self, **kwargs):+ raise RuntimeError("Simulated error")++ student = SimpleModule("input -> output")+ teacher = BuggyModule("input -> output")++ # Setup DummyLM to simulate an error scenario+ lm = DummyLM(+ [+ "Initial thoughts", # Simulate initial teacher's prediction+ ]+ )+ dspy.settings.configure(lm=lm)++ bootstrap = BootstrapFewShot(+ metric=simple_metric,+ max_bootstrapped_demos=1,+ max_labeled_demos=1,+ max_errors=1,+ )++ with pytest.raises(RuntimeError, match="Simulated error"):+ bootstrap.compile(student, teacher=teacher, trainset=trainset, valset=valset)+++def test_validation_set_usage():+ """+ Test to ensure the validation set is correctly used during bootstrapping+ """+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output")++ lm = DummyLM(+ [+ "Initial thoughts",+ "Finish[blue]", # Expected output for both training and validation+ ]+ )+ dspy.settings.configure(lm=lm)++ bootstrap = BootstrapFewShot(+ metric=simple_metric, max_bootstrapped_demos=1, max_labeled_demos=1+ )+ compiled_student = bootstrap.compile(+ student, teacher=teacher, trainset=trainset, valset=valset+ )++ # Check that validation examples are part of student's demos after compilation+ assert len(compiled_student.predictor.demos) >= len(+ valset+ ), "Validation set not used in compiled student demos"diff --git a/tests/teleprompt/test_ensemble.py b/tests/teleprompt/test_ensemble.pynew file mode 100644index 000000000..292176af4--- /dev/null+++ b/tests/teleprompt/test_ensemble.py@@ -0,0 +1,60 @@+import pytest+import dspy+from dspy.teleprompt.ensemble import Ensemble+++class MockProgram(dspy.Module):+ def __init__(self, output):+ super().__init__()+ self.output = output++ def forward(self, *args, **kwargs):+ return self.output+++# Simple reduction function to test with+def mock_reduce_fn(outputs):+ return sum(outputs) / len(outputs)+++def test_ensemble_without_reduction():+ """Test that Ensemble correctly combines outputs without applying a reduce_fn."""+ programs = [MockProgram(i) for i in range(5)]+ ensemble = Ensemble()+ ensembled_program = ensemble.compile(programs)++ outputs = ensembled_program()+ assert len(outputs) == 5, "Ensemble did not combine the correct number of outputs"+++def test_ensemble_with_reduction():+ """Test that Ensemble correctly applies a reduce_fn to combine outputs."""+ programs = [MockProgram(i) for i in range(5)]+ ensemble = Ensemble(reduce_fn=mock_reduce_fn)+ ensembled_program = ensemble.compile(programs)++ output = ensembled_program()+ expected_output = sum(range(5)) / 5+ assert output == expected_output, "Ensemble did not correctly apply the reduce_fn"+++def test_ensemble_with_size_limitation():+ """Test that specifying a size limits the number of programs used in the ensemble."""+ programs = [MockProgram(i) for i in range(10)]+ ensemble_size = 3+ ensemble = Ensemble(size=ensemble_size)+ ensembled_program = ensemble.compile(programs)++ outputs = ensembled_program()+ assert (+ len(outputs) == ensemble_size+ ), "Ensemble did not respect the specified size limitation"+++def test_ensemble_deterministic_behavior():+ """Verify that the Ensemble class raises an assertion for deterministic behavior."""+ with pytest.raises(+ AssertionError,+ match="TODO: Implement example hashing for deterministic ensemble.",+ ):+ Ensemble(deterministic=True)diff --git a/tests/teleprompt/test_finetune.py b/tests/teleprompt/test_finetune.pynew file mode 100644index 000000000..f87f5c14c--- /dev/null+++ b/tests/teleprompt/test_finetune.py@@ -0,0 +1 @@+# TODO\ No newline at end of filediff --git a/tests/teleprompt/test_knn_fewshot.py b/tests/teleprompt/test_knn_fewshot.pynew file mode 100644index 000000000..b267d3dce--- /dev/null+++ b/tests/teleprompt/test_knn_fewshot.py@@ -0,0 +1,72 @@+import pytest+import dsp, dspy+from dspy.predict.knn import KNN+from dspy.teleprompt.knn_fewshot import KNNFewShot+from dspy.utils.dummies import DummyLM, DummyVectorizer+++def mock_example(question: str, answer: str) -> dsp.Example:+ """Creates a mock DSP example with specified question and answer."""+ return dspy.Example(question=question, answer=answer).with_inputs("question")+++def setup_knn_few_shot():+ """Sets up a KNNFewShot instance for testing."""+ trainset = [+ mock_example("What is the capital of France?", "Paris"),+ mock_example("What is the largest ocean?", "Pacific"),+ mock_example("What is 2+2?", "4"),+ ]+ dsp.SentenceTransformersVectorizer = DummyVectorizer+ knn_few_shot = KNNFewShot(KNN, k=2, trainset=trainset)+ return knn_few_shot+++def test_knn_few_shot_initialization(setup_knn_few_shot):+ """Tests the KNNFewShot initialization."""+ knn_few_shot = setup_knn_few_shot+ assert knn_few_shot.KNN.k == 2, "Incorrect k value for KNN"+ assert len(knn_few_shot.KNN.trainset) == 3, "Incorrect trainset size for KNN"+++class SimpleModule(dspy.Module):+ def __init__(self, signature):+ super().__init__()+ self.predictor = dspy.Predict(signature)++ def forward(self, *args, **kwargs):+ return self.predictor(**kwargs)++ def reset_copy(self):+ # Creates a new instance of SimpleModule with the same predictor+ return SimpleModule(self.predictor.signature)+++# TODO: Test not working yet+def _test_knn_few_shot_compile(setup_knn_few_shot):+ """Tests the compile method of KNNFewShot with SimpleModule as student."""+ student = SimpleModule("input -> output")+ teacher = SimpleModule("input -> output") # Assuming teacher uses the same module type++ # Setup DummyLM with a response for a query similar to one of the training examples+ lm = DummyLM(["Madrid", "10"])+ dspy.settings.configure(lm=lm) # Responses for the capital of Spain and the result of 5+5)++ knn_few_shot = setup_knn_few_shot+ trainset = knn_few_shot.KNN.trainset+ compiled_student = knn_few_shot.compile(student, teacher=teacher, trainset=trainset, valset=None)++ assert len(compiled_student.predictor.demos) == 1+ assert compiled_student.predictor.demos[0].input == trainset[0].input+ assert compiled_student.predictor.demos[0].output == trainset[0].output++ # Simulate a query that is similar to one of the training examples+ output = compiled_student.forward(input = "What is the capital of Spain?").output++ print("CONVO")+ print(lm.get_convo(-1))++ # Validate that the output corresponds to one of the expected DummyLM responses+ # This assumes the compiled_student's forward method will execute the predictor with the given query+ assert output in ["Madrid", "10"], "The compiled student did not return the correct output based on the query"diff --git a/tests/teleprompt/test_signature_opt.py b/tests/teleprompt/test_signature_opt.pynew file mode 100644index 000000000..d7f347551--- /dev/null+++ b/tests/teleprompt/test_signature_opt.py@@ -0,0 +1,121 @@+import textwrap+import dspy+from dspy.teleprompt.signature_opt import SignatureOptimizer+from dspy.utils.dummies import DummyLM+from dspy import Example++# Define a simple metric function for testing+def simple_metric(example, prediction):+ # Simplified metric for testing: true if prediction matches expected output+ return example.output == prediction.output++# Example training and validation sets+trainset = [+ Example(input="Question: What is the color of the sky?", output="blue").with_inputs("input"),+ Example(input="Question: What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!").with_inputs("input"),+]++def test_signature_optimizer_initialization():+ optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ assert optimizer.metric == simple_metric, "Metric not correctly initialized"+ assert optimizer.breadth == 2, "Breadth not correctly initialized"+ assert optimizer.depth == 1, "Depth not correctly initialized"+ assert optimizer.init_temperature == 1.4, "Initial temperature not correctly initialized"++class SimpleModule(dspy.Module):+ def __init__(self, signature):+ super().__init__()+ # SignatureOptimizer doesn't work with dspy.Predict+ self.predictor = dspy.ChainOfThought(signature)++ def forward(self, **kwargs):+ return self.predictor(**kwargs)++def test_signature_optimizer_optimization_process():+ optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ dspy.settings.configure(lm=DummyLM(["Optimized instruction 1", "Optimized instruction 2"]))++ student = SimpleModule("input -> output")++ # Assuming the compile method of SignatureOptimizer requires a student module, a development set, and evaluation kwargs+ optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})++ # Check that the optimized student has been modified from the original+ # This check can be more specific based on how the optimization modifies the student+ assert optimized_student is not student, "Optimization did not modify the student"++ # Further tests can be added to verify the specifics of the optimization process,+ # such as checking the instructions of the optimized student's predictors.++def test_signature_optimizer_statistics_tracking():+ optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer.track_stats = True # Enable statistics tracking++ dspy.settings.configure(lm=DummyLM(["Optimized instruction"]))+ student = SimpleModule("input -> output")+ optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})++ # Verify that statistics have been tracked and attached to the optimized student+ assert hasattr(optimized_student, 'total_calls'), "Total calls statistic not tracked"+ assert hasattr(optimized_student, 'results_best'), "Best results statistics not tracked"++# Assuming the setup_signature_optimizer fixture and simple_metric function are defined as before++def test_optimization_and_output_verification():+ lm = DummyLM([+ "Optimized Prompt",+ "Optimized Prefix",+ ])+ dspy.settings.configure(lm=lm)+ optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)++ student = SimpleModule("input -> output")++ # Compile the student with the optimizer+ optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})++ # Simulate calling the optimized student with a new input+ test_input = "What is the capital of France?"+ prediction = optimized_student(input=test_input)++ print(lm.get_convo(-1))++ assert prediction.output == "No more responses"++ assert lm.get_convo(-1) == textwrap.dedent("""\+ Optimized Prompt++ ---++ Follow the following format.++ Input: ${input}+ Reasoning: Let's think step by step in order to ${produce the output}. We ...+ Optimized Prefix ${output}++ ---++ Input: What is the capital of France?+ Reasoning: Let's think step by step in order to No more responses+ Optimized Prefix No more responses""")++def test_statistics_tracking_during_optimization():+ dspy.settings.configure(lm=DummyLM(["Optimized instruction for stats tracking"]))++ optimizer = SignatureOptimizer(metric=simple_metric, breadth=2, depth=1, init_temperature=1.4)+ optimizer.track_stats = True # Enable statistics tracking++ student = SimpleModule("input -> output")+ optimized_student = optimizer.compile(student, devset=trainset, eval_kwargs={"num_threads": 1, "display_progress": False})++ # Verify that statistics have been tracked+ assert hasattr(optimized_student, 'total_calls'), "Optimizer did not track total metric calls"+ assert optimized_student.total_calls > 0, "Optimizer reported no metric calls"++ # Check if the results_best and results_latest contain valid statistics+ assert 'results_best' in optimized_student.__dict__, "Optimizer did not track the best results"+ assert 'results_latest' in optimized_student.__dict__, "Optimizer did not track the latest results"+ assert len(optimized_student.results_best) > 0, "Optimizer did not properly populate the best results statistics"+ assert len(optimized_student.results_latest) > 0, "Optimizer did not properly populate the latest results statistics"++ # Additional detailed checks can be added here to verify the contents of the tracked statisticsdiff --git a/tests/teleprompt/test_signature_opt_bayesian.py b/tests/teleprompt/test_signature_opt_bayesian.pynew file mode 100644index 000000000..abbcf11d1--- /dev/null+++ b/tests/teleprompt/test_signature_opt_bayesian.py@@ -0,0 +1,176 @@+import textwrap+import pytest+import dspy+from dspy.teleprompt.signature_opt_bayesian import BayesianSignatureOptimizer+from dspy.utils.dummies import DummyLM+from dspy import Example+++# Define a simple metric function for testing+def simple_metric(example, prediction, trace=None):+ # Simplified metric for testing: true if prediction matches expected output+ return example.output == prediction.output+++# Example training and validation sets+trainset = [+ Example(input="What is the color of the sky?", output="blue").with_inputs("input"),+ Example(+ input="What does the fox say?", output="Ring-ding-ding-ding-dingeringeding!"+ ).with_inputs("input"),+ Example(input="What is the capital of France?", output="Paris").with_inputs(+ "input"+ ),+ Example(input="What is the capital of Germany?", output="Berlin").with_inputs(+ "input"+ ),+]+++def test_bayesian_signature_optimizer_initialization():+ optimizer = BayesianSignatureOptimizer(+ metric=simple_metric, n=10, init_temperature=1.4, verbose=True, track_stats=True+ )+ assert optimizer.metric == simple_metric, "Metric not correctly initialized"+ assert optimizer.n == 10, "Incorrect 'n' parameter initialization"+ assert (+ optimizer.init_temperature == 1.4+ ), "Initial temperature not correctly initialized"+ assert optimizer.verbose is True, "Verbose flag not correctly initialized"+ assert optimizer.track_stats is True, "Track stats flag not correctly initialized"+++class SimpleModule(dspy.Module):+ def __init__(self, signature):+ super().__init__()+ # SignatureOptimizer doesn't work with dspy.Predict+ self.predictor = dspy.ChainOfThought(signature)++ def forward(self, **kwargs):+ return self.predictor(**kwargs)+++def test_signature_optimizer_optimization_process():+ # Make LM that is always right+ dspy.settings.configure(lm=DummyLM({ex.input: ex.output for ex in trainset}))++ student = SimpleModule(signature="input -> output")++ optimizer = BayesianSignatureOptimizer(+ metric=simple_metric,+ n=10,+ init_temperature=1.4,+ verbose=False,+ track_stats=False,+ )++ # Adjustments: Include required parameters for the compile method+ optimized_student = optimizer.compile(+ student=student,+ devset=trainset,+ optuna_trials_num=10,+ max_bootstrapped_demos=3,+ max_labeled_demos=5,+ eval_kwargs={"num_threads": 1, "display_progress": False},+ )++ assert len(optimized_student.predictor.demos) == 4+++def test_signature_optimizer_bad_lm():+ dspy.settings.configure(+ lm=DummyLM([f"Optimized instruction {i}" for i in range(30)])+ )+ student = SimpleModule(signature="input -> output")+ optimizer = BayesianSignatureOptimizer(+ metric=simple_metric,+ n=10,+ init_temperature=1.4,+ verbose=False,+ track_stats=False,+ )++ # Krista: when the code tries to generate bootstrapped examples, the examples are generated using DummyLM,+ # which only outputs "Optimized instruction i" this means that none of the bootstrapped examples are successful,+ # and therefore the set of examples that we're using to generate new prompts is empty+ with pytest.raises(ValueError):+ _optimized_student = optimizer.compile(+ student=student,+ devset=trainset,+ optuna_trials_num=10,+ max_bootstrapped_demos=3,+ max_labeled_demos=5,+ eval_kwargs={"num_threads": 1, "display_progress": False},+ )+++def test_optimization_and_output_verification():+ # Make a language model that is always right, except on the last+ # example in the train set.+ lm = DummyLM({ex.input: ex.output for ex in trainset[:-1]}, follow_examples=True)+ dspy.settings.configure(lm=lm)++ optimizer = BayesianSignatureOptimizer(+ metric=simple_metric,+ n=10,+ init_temperature=1.4,+ verbose=False,+ track_stats=True,+ )++ student = SimpleModule("input -> output")++ # Compile the student with the optimizer+ optimized_student = optimizer.compile(+ student=student,+ devset=trainset,+ optuna_trials_num=10,+ max_bootstrapped_demos=3,+ max_labeled_demos=5,+ eval_kwargs={"num_threads": 1, "display_progress": False},+ )++ # Simulate calling the optimized student with a new input+ test_input = "What is the capital of France?"+ prediction = optimized_student(input=test_input)++ print("CORRECT ANSWER")+ print(lm.get_convo(-1))++ assert prediction.output == "blue"++ assert lm.get_convo(-1) == textwrap.dedent(+ """\+ Given the fields `input`, produce the fields `output`.++ ---++ Input: What does the fox say?+ Output: Ring-ding-ding-ding-dingeringeding!++ Input: What is the capital of Germany?+ Output: Berlin++ Input: What is the capital of France?+ Output: Paris++ ---++ Follow the following format.++ Input: ${input}+ Reasoning: Let's think step by step in order to ${produce the output}. We ...+ Output: ${output}++ ---++ Input: What is the color of the sky?+ Reasoning: Let's think step by step in order to blue+ Output: blue++ ---++ Input: What is the capital of France?+ Reasoning: Let's think step by step in order to blue+ Output: blue"""+ )