Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Cleaned up functional

Thomas D Ahle0bf81c82e9b5
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index 394d67667..6fe342333 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -137,6 +137,8 @@ class TypedPredictor(dspy.Module):
def forward(self, **kwargs):
modified_kwargs = kwargs.copy()
+ # We have to re-prepare the signature on every forward call, because the base
+ # signature might have been modified by an optimizer or something like that.
signature = self._prepare_signature()
for try_i in range(MAX_RETRIES):
result = self.predictor(**modified_kwargs, new_signature=signature)
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index b73c956d7..c0a56070c 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -1,6 +1,7 @@
from copy import deepcopy
import dsp
from pydantic import BaseModel, Field, create_model
+from pydantic.fields import FieldInfo
from typing import Type, Union, Dict, Tuple
import re
@@ -68,9 +69,7 @@ class SignatureMeta(type(BaseModel)):
return getattr(cls, "__doc__", "")
def with_instructions(cls, instructions: str):
- return create_model(
- cls.__name__, __base__=Signature, __doc__=instructions, **cls.fields
- )
+ return Signature(cls.fields, instructions)
@property
def fields(cls):
@@ -85,9 +84,7 @@ class SignatureMeta(type(BaseModel)):
**fields_copy[name].json_schema_extra,
**kwargs,
}
- return create_model(
- cls.__name__, __base__=Signature, __doc__=cls.instructions, **fields_copy
- )
+ return Signature(fields_copy, cls.instructions)
@property
def input_fields(cls):
@@ -113,8 +110,10 @@ class SignatureMeta(type(BaseModel)):
def insert(cls, index: int, name: str, field, type_: Type = None):
# It's posisble to set the type as annotation=type in pydantic.Field(...)
# But this may be annoying for users, so we allow them to pass the type
- if type_ is not None:
- field.annotation = type_
+ if type_ is None:
+ type_ = field.annotation
+ if type_ is None:
+ type_ = str
input_fields = list(cls.input_fields.items())
output_fields = list(cls.output_fields.items())
@@ -130,14 +129,10 @@ class SignatureMeta(type(BaseModel)):
index += len(lst) + 1
if index < 0 or index > len(lst):
raise ValueError(f"Invalid index: {index}")
- lst.insert(index, (name, field))
+ lst.insert(index, (name, (type_, field)))
new_fields = dict(input_fields + output_fields)
- new_signature = create_model(
- cls.__name__ + "'", __base__=Signature, **new_fields
- )
- new_signature.__doc__ = cls.instructions
- return new_signature
+ return Signature(new_fields, cls.instructions)
def _parse_signature(cls, signature: str) -> Tuple[Type, Field]:
pattern = r"^\s*[\w\s,]+\s*->\s*[\w\s,]+\s*$"
@@ -157,7 +152,7 @@ class SignatureMeta(type(BaseModel)):
def __call__(
cls,
- signature: Union[str, Dict[str, Tuple[type, Field]]],
+ signature: Union[str, Dict[str, Tuple[type, FieldInfo]]],
instructions: str = None,
):
"""
@@ -175,13 +170,35 @@ class SignatureMeta(type(BaseModel)):
fields = cls._parse_signature(signature)
else:
fields = signature
+
+ # Validate the fields, this is important because we sometimes forget the
+ # slightly unintuitive syntax with tuples of (type, Field)
+ fixed_fields = {}
+ for name, type_field in fields.items():
+ assert isinstance(name, str), f"Field names must be strings, not {type(name)}"
+ if isinstance(type_field, FieldInfo):
+ type_ = type_field.annotation
+ field = type_field
+ else:
+ assert isinstance(type_field, tuple), f"Field values must be tuples, not {type(type_field)}"
+ type_, field = type_field
+ # It might be better to be explicit about the type, but it currently would break
+ # program of thought and teleprompters, so we just silently default to string.
+ if type_ is None:
+ type_ = str
+ assert isinstance(type_, type), f"Field types must be types, not {type(type_)}"
+ assert isinstance(field, FieldInfo), f"Field values must be Field instances, not {type(field)}"
+ fixed_fields[name] = (type_, field)
+
+ # Fixing the fields shouldn't change the order
+ assert list(fixed_fields.keys()) == list(fields.keys())
# Default prompt when no instructions are provided
if instructions is None:
sig = Signature(signature, "") # Simple way to parse input/output fields
instructions = _default_instructions(sig)
- signature = create_model("Signature", __base__=Signature, **fields)
+ signature = create_model("Signature", __base__=Signature, **fixed_fields)
signature.__doc__ = instructions
return signature
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index f5b8eecdd..8605cd427 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -93,7 +93,7 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
program2 = teleprompter.compile(student, teacher=teacher, trainset=trainset2)
evaluate = Evaluate(devset=self.valset, metric=self.metric, num_threads=self.num_threads,
- display_table=False, display_progress=True, max_errors=100)
+ display_table=False, display_progress=True)
score, subscores = evaluate(program2, return_all_scores=True)
diff --git a/dspy/teleprompt/signature_opt_llmo b/dspy/teleprompt/signature_opt_llmo
deleted file mode 100644
index 152777f6f..000000000
--- a/dspy/teleprompt/signature_opt_llmo
+++ /dev/null
@@ -1,10 +0,0 @@
-from dspy.teleprompt.teleprompt import Teleprompter
-
-class SignatureOptimizerLLMO(Teleprompter):
- def __init__(self, evaluate):
- super().__init__()
- self.evaluate = evaluate
-
- def compile(self, module):
- for predictor in module.predictors():
- predictor.signature
\ No newline at end of file
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 8adf6cd16..e85c19f2d 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -9,7 +9,7 @@ import warnings
import pytest
import dspy
-from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor
+from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor, functional
from dspy.primitives.example import Example
from dspy.teleprompt.bootstrap import BootstrapFewShot
from dspy.utils.dummies import DummyLM
@@ -76,7 +76,7 @@ def test_simple_class():
description="At least two comments about the answer"
)
- class QA(dspy.Module):
+ class QA(FunctionalModule):
@predictor
def hard_question(self, topic: str) -> str:
"""Think of a hard factual question about a topic. It should be answerable with a number."""
@@ -107,8 +107,13 @@ def test_simple_class():
dspy.settings.configure(lm=lm)
qa = QA()
+ assert isinstance(qa, FunctionalModule)
+ assert isinstance(qa.answer, functional._StripOutput)
+
question, answer = qa(topic="Physics")
+ print(qa.answer)
+
assert question == "What is the speed of light?"
assert answer == expected
@@ -168,7 +173,7 @@ def test_named_params():
named_predictors = list(qa.named_predictors())
assert len(named_predictors) == 2
names, _ = zip(*qa.named_predictors())
- assert set(names) == {"hard_question.predictor", "answer.predictor"}
+ assert set(names) == {"hard_question.predictor.predictor", "answer.predictor.predictor"}
def test_bootstrap_effectiveness():
@@ -215,9 +220,9 @@ def test_bootstrap_effectiveness():
lm.inspect_history(n=2)
# Check that the compiled student has the correct demos
- assert len(compiled_student.output.predictor.demos) == 1
- assert compiled_student.output.predictor.demos[0].input == trainset[0].input
- assert compiled_student.output.predictor.demos[0].output == trainset[0].output
+ assert len(compiled_student.output.predictor.predictor.demos) == 1
+ assert compiled_student.output.predictor.predictor.demos[0].input == trainset[0].input
+ assert compiled_student.output.predictor.predictor.demos[0].output == trainset[0].output
# Test the compiled student's prediction.
# We are using a DummyLM with follow_examples=True, which means that
diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.py
index b5e62425d..c28fb375f 100644
--- a/tests/predict/test_chain_of_thought_with_hint.py
+++ b/tests/predict/test_chain_of_thought_with_hint.py
@@ -7,6 +7,7 @@ def test_cot_with_no_hint():
lm = DummyLM(["find the number after 1", "2"])
dspy.settings.configure(lm=lm)
predict = ChainOfThoughtWithHint("question -> answer")
+ # Check output fields have the right order
assert list(predict.extended_signature2.output_fields.keys()) == [
"rationale",
"hint",