DSPy: The framework for programming—not prompting—language models
Python
Fixed bug with wrapped output fields in examples
Thomas D Ahle702a89c443de
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.pyindex 3003e9eef..f349fa2e4 100644--- a/dspy/functional/functional.py+++ b/dspy/functional/functional.py@@ -1,5 +1,5 @@import inspect, os, openai, dspy, typing, pydantic-from typing import Annotated+from typing import Annotated, List, Tupleimport typingfrom dsp.templates import passages2textimport json@@ -27,7 +27,7 @@ class _StripOutput(dspy.Module):super().__init__()self.predictor = predictorself.output_key = output_key-+def copy(self):return _StripOutput(self.predictor.copy(), self.output_key)@@ -37,7 +37,8 @@ class _StripOutput(dspy.Module):class FunctionalModule(dspy.Module):- """ To use the @cot and @predictor decorators, your module needs to inheret form this class. """+ """To use the @cot and @predictor decorators, your module needs to inheret form this class."""+def __init__(self):super().__init__()for name in dir(self):@@ -47,16 +48,18 @@ class FunctionalModule(dspy.Module):def TypedChainOfThought(signature):- """ Just like TypedPredictor, but adds a ChainOfThought OutputField. """+ """Just like TypedPredictor, but adds a ChainOfThought OutputField."""signature = ensure_signature(signature)output_keys = ", ".join(signature.output_fields.keys())- return TypedPredictor(signature.prepend(- "reasoning",- dspy.OutputField(- prefix="Reasoning: Let's think step by step in order to",- desc="${produce the " + output_keys + "}. We ...",- ),- ))+ return TypedPredictor(+ signature.prepend(+ "reasoning",+ dspy.OutputField(+ prefix="Reasoning: Let's think step by step in order to",+ desc="${produce the " + output_keys + "}. We ...",+ ),+ )+ )class TypedPredictor(dspy.Module):@@ -108,28 +111,25 @@ class TypedPredictor(dspy.Module):else:# Anything else we wrap in a pydantic objectunwrap = lambda x: x+ wrap = lambda x: xif not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):- type_ = pydantic.create_model(- "Output", value=(type_, ...), __base__=pydantic.BaseModel- )+ type_ = pydantic.create_model("Output", value=(type_, ...), __base__=pydantic.BaseModel)+ wrap = lambda x: type_(value=x)unwrap = lambda x: x.valuesignature = signature.with_updated_fields(name,desc=field.json_schema_extra.get("desc", "")+ (- f". Respond with a single JSON object using the schema "+ f". Respond with a single JSON object. JSON Schema: "+ json.dumps(type_.model_json_schema())),- format=lambda x: (- x if isinstance(x, str) else x.model_dump_json()- ),- parser=lambda x: unwrap(- type_.model_validate_json(_unwrap_json(x))- ),+ format=lambda x: (x if isinstance(x, str) else wrap(x).model_dump_json()),+ parser=lambda x: unwrap(type_.model_validate_json(_unwrap_json(x))),+ type_=type_,)else: # If input fieldformat = lambda x: x if isinstance(x, str) else str(x)- if type_ in (list[str], tuple[str]):+ if type_ in (List[str], list[str], Tuple[str], tuple[str]):format = passages2textelif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):format = lambda x: x if isinstance(x, str) else x.model_dump_json()@@ -156,15 +156,15 @@ class TypedPredictor(dspy.Module):errors[name] = _format_error(e)# If we can, we add an example to the error messagecurrent_desc = field.json_schema_extra.get("desc", "")- i = current_desc.find("Respond with a single JSON object using the schema")+ i = current_desc.find("JSON Schema: ")if i == -1:continue # Only add examples to JSON objectssuffix, current_desc = current_desc[i:], current_desc[:i]prefix = "You MUST use this format: "if try_i + 1 < MAX_RETRIES and prefix not in current_desc:if example := self._make_example(field.annotation):- signature = signature.with_updated_fields(name,- desc = current_desc + "\n" + prefix + example + "\n" + suffix+ signature = signature.with_updated_fields(+ name, desc=current_desc + "\n" + prefix + example + "\n" + suffix)if errors:# Add new fields for each error@@ -173,8 +173,7 @@ class TypedPredictor(dspy.Module):signature = signature.append(f"error_{name}_{try_i}",dspy.InputField(- prefix=f"Past Error "- + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),+ prefix=f"Past Error " + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),desc="An error to avoid in the future",),)@@ -183,8 +182,9 @@ class TypedPredictor(dspy.Module):for name, value in parsed_results.items():setattr(result, name, value)return result- raise ValueError("Too many retries trying to get the correct output format. "- + "Try simplifying the requirements.")+ raise ValueError(+ "Too many retries trying to get the correct output format. " + "Try simplifying the requirements.", errors+ )def _format_error(error: Exception):@@ -249,9 +249,7 @@ def main():class Answer(pydantic.BaseModel):value: floatcertainty: float- comments: list[str] = pydantic.Field(- description="At least two comments about the answer"- )+ comments: list[str] = pydantic.Field(description="At least two comments about the answer")class QA(dspy.Module):@predictor@@ -290,16 +288,11 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):if not dspy.evaluate.answer_passage_match(example, pred):return False- hops = [example.question] + [- outputs.query for *_, outputs in trace if "query" in outputs- ]+ hops = [example.question] + [outputs.query for *_, outputs in trace if "query" in outputs]if max([len(h) for h in hops]) > 100:return False- if any(- dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)- for idx in range(2, len(hops))- ):+ if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))):return Falsereturn True@@ -307,9 +300,7 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):def gold_passages_retrieved(example, pred, trace=None):gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))- found_titles = set(- map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context])- )+ found_titles = set(map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context]))return gold_titles.issubset(found_titles)@@ -322,9 +313,7 @@ def hotpot():from dspy.teleprompt.bootstrap import BootstrapFewShotprint("Load the dataset.")- dataset = HotPotQA(- train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0- )+ dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)trainset = [x.with_inputs("question") for x in dataset.train]devset = [x.with_inputs("question") for x in dataset.dev]print("Done")@@ -361,9 +350,7 @@ def hotpot():lm = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=4000)dspy.settings.configure(lm=lm, rm=rm, trace=[])- evaluate_on_hotpotqa = Evaluate(- devset=devset, num_threads=10, display_progress=True, display_table=5- )+ evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=5)# uncompiled (i.e., zero-shot) programuncompiled_baleen = SimplifiedBaleen()@@ -373,9 +360,7 @@ def hotpot():)# compiled (i.e., few-shot) program- compiled_baleen = BootstrapFewShot(- metric=validate_context_and_answer_and_hops- ).compile(+ compiled_baleen = BootstrapFewShot(metric=validate_context_and_answer_and_hops).compile(SimplifiedBaleen(),teacher=SimplifiedBaleen(passages_per_hop=2),trainset=trainset,diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.pyindex ac9a334ce..a5f6b976a 100644--- a/dspy/signatures/signature.py+++ b/dspy/signatures/signature.py@@ -77,7 +77,7 @@ class SignatureMeta(type(BaseModel)):# Make sure to give input fields before output fieldsreturn {**cls.input_fields, **cls.output_fields}- def with_updated_fields(cls, name, **kwargs):+ def with_updated_fields(cls, name, type_=None, **kwargs):"""Returns a new Signature type with the field, name, updatedwith fields[name].json_schema_extra[key] = value."""fields_copy = deepcopy(cls.fields)@@ -85,6 +85,8 @@ class SignatureMeta(type(BaseModel)):**fields_copy[name].json_schema_extra,**kwargs,}+ if type_ is not None:+ fields_copy[name].annotation = type_return Signature(fields_copy, cls.instructions)@propertydiff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.pyindex 216ec3315..5d0f230b1 100644--- a/tests/functional/test_functional.py+++ b/tests/functional/test_functional.py@@ -31,8 +31,8 @@ def test_simple():def test_list_output():@predictor- def hard_question(topic: str) -> List[str]:- """Think of a hard factual question about a topic."""+ def hard_questions(topics: List[str]) -> List[str]:+ passexpected = ["What is the speed of light?", "What is the speed of sound?"]lm = DummyLM(@@ -40,7 +40,7 @@ def test_list_output():)dspy.settings.configure(lm=lm)- question = hard_question(topic="Physics")+ question = hard_questions(topics=["Physics", "Music"])lm.inspect_history(n=2)assert question == expected@@ -369,7 +369,7 @@ def test_multi_errors():Past Error (flight_information, 2): An error to avoid in the future- Flight Information: ${flight_information}. Respond with a single JSON object using the schema {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}+ Flight Information: ${flight_information}. Respond with a single JSON object. JSON Schema: {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}---@@ -422,7 +422,7 @@ def test_field_validator():Past Error (get_user_details): An error to avoid in the futurePast Error (get_user_details, 2): An error to avoid in the future- Get User Details: ${get_user_details}. Respond with a single JSON object using the schema {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}+ Get User Details: ${get_user_details}. Respond with a single JSON object. JSON Schema: {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}---@@ -430,3 +430,20 @@ def test_field_validator():Past Error (get_user_details, 2): Value error, Name must be in uppercase.: name (error type: value_error)Get User Details: {"name": "lower case name", "age": 25}""")+++def test_annotated_field():+ # Since we don't currently validate fields on the main signature,+ # the annotated fields are also not validated.+ # But at least it should not crash.++ @predictor+ def test(input: Annotated[str, Field(description="description")]) -> Annotated[float, Field(gt=0, lt=1)]:+ pass++ lm = DummyLM(["0.5"])+ dspy.settings.configure(lm=lm)++ output = test(input="input")++ assert output == 0.5