Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Fixed bug with wrapped output fields in examples

Thomas D Ahle702a89c443de
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index 3003e9eef..f349fa2e4 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -1,5 +1,5 @@
import inspect, os, openai, dspy, typing, pydantic
-from typing import Annotated
+from typing import Annotated, List, Tuple
import typing
from dsp.templates import passages2text
import json
@@ -27,7 +27,7 @@ class _StripOutput(dspy.Module):
super().__init__()
self.predictor = predictor
self.output_key = output_key
-
+
def copy(self):
return _StripOutput(self.predictor.copy(), self.output_key)
@@ -37,7 +37,8 @@ class _StripOutput(dspy.Module):
class FunctionalModule(dspy.Module):
- """ To use the @cot and @predictor decorators, your module needs to inheret form this class. """
+ """To use the @cot and @predictor decorators, your module needs to inheret form this class."""
+
def __init__(self):
super().__init__()
for name in dir(self):
@@ -47,16 +48,18 @@ class FunctionalModule(dspy.Module):
def TypedChainOfThought(signature):
- """ Just like TypedPredictor, but adds a ChainOfThought OutputField. """
+ """Just like TypedPredictor, but adds a ChainOfThought OutputField."""
signature = ensure_signature(signature)
output_keys = ", ".join(signature.output_fields.keys())
- return TypedPredictor(signature.prepend(
- "reasoning",
- dspy.OutputField(
- prefix="Reasoning: Let's think step by step in order to",
- desc="${produce the " + output_keys + "}. We ...",
- ),
- ))
+ return TypedPredictor(
+ signature.prepend(
+ "reasoning",
+ dspy.OutputField(
+ prefix="Reasoning: Let's think step by step in order to",
+ desc="${produce the " + output_keys + "}. We ...",
+ ),
+ )
+ )
class TypedPredictor(dspy.Module):
@@ -108,28 +111,25 @@ class TypedPredictor(dspy.Module):
else:
# Anything else we wrap in a pydantic object
unwrap = lambda x: x
+ wrap = lambda x: x
if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):
- type_ = pydantic.create_model(
- "Output", value=(type_, ...), __base__=pydantic.BaseModel
- )
+ type_ = pydantic.create_model("Output", value=(type_, ...), __base__=pydantic.BaseModel)
+ wrap = lambda x: type_(value=x)
unwrap = lambda x: x.value
signature = signature.with_updated_fields(
name,
desc=field.json_schema_extra.get("desc", "")
+ (
- f". Respond with a single JSON object using the schema "
+ f". Respond with a single JSON object. JSON Schema: "
+ json.dumps(type_.model_json_schema())
),
- format=lambda x: (
- x if isinstance(x, str) else x.model_dump_json()
- ),
- parser=lambda x: unwrap(
- type_.model_validate_json(_unwrap_json(x))
- ),
+ format=lambda x: (x if isinstance(x, str) else wrap(x).model_dump_json()),
+ parser=lambda x: unwrap(type_.model_validate_json(_unwrap_json(x))),
+ type_=type_,
)
else: # If input field
format = lambda x: x if isinstance(x, str) else str(x)
- if type_ in (list[str], tuple[str]):
+ if type_ in (List[str], list[str], Tuple[str], tuple[str]):
format = passages2text
elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):
format = lambda x: x if isinstance(x, str) else x.model_dump_json()
@@ -156,15 +156,15 @@ class TypedPredictor(dspy.Module):
errors[name] = _format_error(e)
# If we can, we add an example to the error message
current_desc = field.json_schema_extra.get("desc", "")
- i = current_desc.find("Respond with a single JSON object using the schema")
+ i = current_desc.find("JSON Schema: ")
if i == -1:
continue # Only add examples to JSON objects
suffix, current_desc = current_desc[i:], current_desc[:i]
prefix = "You MUST use this format: "
if try_i + 1 < MAX_RETRIES and prefix not in current_desc:
if example := self._make_example(field.annotation):
- signature = signature.with_updated_fields(name,
- desc = current_desc + "\n" + prefix + example + "\n" + suffix
+ signature = signature.with_updated_fields(
+ name, desc=current_desc + "\n" + prefix + example + "\n" + suffix
)
if errors:
# Add new fields for each error
@@ -173,8 +173,7 @@ class TypedPredictor(dspy.Module):
signature = signature.append(
f"error_{name}_{try_i}",
dspy.InputField(
- prefix=f"Past Error "
- + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),
+ prefix=f"Past Error " + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),
desc="An error to avoid in the future",
),
)
@@ -183,8 +182,9 @@ class TypedPredictor(dspy.Module):
for name, value in parsed_results.items():
setattr(result, name, value)
return result
- raise ValueError("Too many retries trying to get the correct output format. "
- + "Try simplifying the requirements.")
+ raise ValueError(
+ "Too many retries trying to get the correct output format. " + "Try simplifying the requirements.", errors
+ )
def _format_error(error: Exception):
@@ -249,9 +249,7 @@ def main():
class Answer(pydantic.BaseModel):
value: float
certainty: float
- comments: list[str] = pydantic.Field(
- description="At least two comments about the answer"
- )
+ comments: list[str] = pydantic.Field(description="At least two comments about the answer")
class QA(dspy.Module):
@predictor
@@ -290,16 +288,11 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
if not dspy.evaluate.answer_passage_match(example, pred):
return False
- hops = [example.question] + [
- outputs.query for *_, outputs in trace if "query" in outputs
- ]
+ hops = [example.question] + [outputs.query for *_, outputs in trace if "query" in outputs]
if max([len(h) for h in hops]) > 100:
return False
- if any(
- dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)
- for idx in range(2, len(hops))
- ):
+ if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))):
return False
return True
@@ -307,9 +300,7 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
def gold_passages_retrieved(example, pred, trace=None):
gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))
- found_titles = set(
- map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context])
- )
+ found_titles = set(map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context]))
return gold_titles.issubset(found_titles)
@@ -322,9 +313,7 @@ def hotpot():
from dspy.teleprompt.bootstrap import BootstrapFewShot
print("Load the dataset.")
- dataset = HotPotQA(
- train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0
- )
+ dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
trainset = [x.with_inputs("question") for x in dataset.train]
devset = [x.with_inputs("question") for x in dataset.dev]
print("Done")
@@ -361,9 +350,7 @@ def hotpot():
lm = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=4000)
dspy.settings.configure(lm=lm, rm=rm, trace=[])
- evaluate_on_hotpotqa = Evaluate(
- devset=devset, num_threads=10, display_progress=True, display_table=5
- )
+ evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=5)
# uncompiled (i.e., zero-shot) program
uncompiled_baleen = SimplifiedBaleen()
@@ -373,9 +360,7 @@ def hotpot():
)
# compiled (i.e., few-shot) program
- compiled_baleen = BootstrapFewShot(
- metric=validate_context_and_answer_and_hops
- ).compile(
+ compiled_baleen = BootstrapFewShot(metric=validate_context_and_answer_and_hops).compile(
SimplifiedBaleen(),
teacher=SimplifiedBaleen(passages_per_hop=2),
trainset=trainset,
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index ac9a334ce..a5f6b976a 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -77,7 +77,7 @@ class SignatureMeta(type(BaseModel)):
# Make sure to give input fields before output fields
return {**cls.input_fields, **cls.output_fields}
- def with_updated_fields(cls, name, **kwargs):
+ def with_updated_fields(cls, name, type_=None, **kwargs):
"""Returns a new Signature type with the field, name, updated
with fields[name].json_schema_extra[key] = value."""
fields_copy = deepcopy(cls.fields)
@@ -85,6 +85,8 @@ class SignatureMeta(type(BaseModel)):
**fields_copy[name].json_schema_extra,
**kwargs,
}
+ if type_ is not None:
+ fields_copy[name].annotation = type_
return Signature(fields_copy, cls.instructions)
@property
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 216ec3315..5d0f230b1 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -31,8 +31,8 @@ def test_simple():
def test_list_output():
@predictor
- def hard_question(topic: str) -> List[str]:
- """Think of a hard factual question about a topic."""
+ def hard_questions(topics: List[str]) -> List[str]:
+ pass
expected = ["What is the speed of light?", "What is the speed of sound?"]
lm = DummyLM(
@@ -40,7 +40,7 @@ def test_list_output():
)
dspy.settings.configure(lm=lm)
- question = hard_question(topic="Physics")
+ question = hard_questions(topics=["Physics", "Music"])
lm.inspect_history(n=2)
assert question == expected
@@ -369,7 +369,7 @@ def test_multi_errors():
Past Error (flight_information, 2): An error to avoid in the future
- Flight Information: ${flight_information}. Respond with a single JSON object using the schema {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}
+ Flight Information: ${flight_information}. Respond with a single JSON object. JSON Schema: {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}
---
@@ -422,7 +422,7 @@ def test_field_validator():
Past Error (get_user_details): An error to avoid in the future
Past Error (get_user_details, 2): An error to avoid in the future
- Get User Details: ${get_user_details}. Respond with a single JSON object using the schema {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}
+ Get User Details: ${get_user_details}. Respond with a single JSON object. JSON Schema: {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}
---
@@ -430,3 +430,20 @@ def test_field_validator():
Past Error (get_user_details, 2): Value error, Name must be in uppercase.: name (error type: value_error)
Get User Details: {"name": "lower case name", "age": 25}"""
)
+
+
+def test_annotated_field():
+ # Since we don't currently validate fields on the main signature,
+ # the annotated fields are also not validated.
+ # But at least it should not crash.
+
+ @predictor
+ def test(input: Annotated[str, Field(description="description")]) -> Annotated[float, Field(gt=0, lt=1)]:
+ pass
+
+ lm = DummyLM(["0.5"])
+ dspy.settings.configure(lm=lm)
+
+ output = test(input="input")
+
+ assert output == 0.5