DSPy: The framework for programming—not prompting—language models
Python
adding fixes and refactoring
manishshettym04ae621ca5cd
diff --git a/assert_example.py b/assert_example.pynew file mode 100644index 000000000..bce2b5621--- /dev/null+++ b/assert_example.py@@ -0,0 +1,92 @@+import os++import dspy+from dsp.utils import deduplicate+from dspy.datasets import HotPotQA+from dspy.teleprompt import BootstrapFewShot++# pipeline configs+turbo = dspy.OpenAI(model='gpt-3.5-turbo')+colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')+dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)+++# load dataset+dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)+trainset = [x.with_inputs('question') for x in dataset.train]+devset = [x.with_inputs('question') for x in dataset.dev]+++# signatures of dspy modules+class GenerateAnswer(dspy.Signature):+ """Answer questions with short factoid answers."""++ context = dspy.InputField(desc="may contain relevant facts")+ question = dspy.InputField()+ answer = dspy.OutputField(desc="often between 1 and 5 words")++class GenerateSearchQuery(dspy.Signature):+ """Write a simple search query that will help answer a complex question."""++ context = dspy.InputField(desc="may contain relevant facts")+ question = dspy.InputField()+ query = dspy.OutputField()++# validation logic for verifying traces+def validate_context_and_answer_and_hops(example, pred, trace=None):+ if not dspy.evaluate.answer_exact_match(example, pred):+ return False++ if not dspy.evaluate.answer_passage_match(example, pred):+ return False++ hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]++ if max([len(h) for h in hops]) > 100:+ return False++ if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): # 4+ return False++ return True+++########################## NEW STUFF ##########################+++# declaration of dspy program+class SimplifiedBaleen(dspy.Module):+ def __init__(self, passages_per_hop=2, max_hops=2):+ super().__init__()++ self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]+ self.retrieve = dspy.Retrieve(k=passages_per_hop)+ self.generate_answer = dspy.ChainOfThought(GenerateAnswer)+ self.max_hops = max_hops++ def forward(self, question):+ context = []++ for hop in range(self.max_hops):+ query = self.generate_query[hop](context=context, question=question).query++ # assertion+ dspy.Assert(lambda x: len(x) <= 100, query)++ passages = self.retrieve(query).passages+ context = deduplicate(context + passages)++ pred = self.generate_answer(context=context, question=question)++ return dspy.Prediction(context=context, answer=pred.answer)++# compile dspy program using a teleprompter (optimizer)+teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)++student = SimplifiedBaleen()+teacher = SimplifiedBaleen(passages_per_hop=2)+compiled_baleen = teleprompter.compile(student, teacher=teacher, trainset=trainset)++my_question = "How many storeys are in the castle that David Gregory inherited?"+pred = compiled_baleen(my_question)+turbo.inspect_history(n=3)\ No newline at end of filediff --git a/dspy/__init__.py b/dspy/__init__.pyindex 26ea97644..18b70008d 100644--- a/dspy/__init__.py+++ b/dspy/__init__.py@@ -4,7 +4,6 @@ from .signatures import *from .retrieve import *from .predict import *from .primitives import *-from .assertions import *# from .evaluation import *diff --git a/dspy/assertions/__init__.py b/dspy/assertions/__init__.pydeleted file mode 100644index 6da1aa37e..000000000--- a/dspy/assertions/__init__.py+++ /dev/null@@ -1 +0,0 @@-from .assertions import *diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.pyindex f99f3af2d..b61bf55e1 100644--- a/dspy/primitives/__init__.py+++ b/dspy/primitives/__init__.py@@ -1,3 +1,4 @@from .example import *from .program import *-from .prediction import *\ No newline at end of file+from .prediction import *+from .assertions import *\ No newline at end of filediff --git a/dspy/assertions/assertions.py b/dspy/primitives/assertions.pysimilarity index 79%rename from dspy/assertions/assertions.pyrename to dspy/primitives/assertions.pyindex ef2f04143..5f1376a29 100644--- a/dspy/assertions/assertions.py+++ b/dspy/primitives/assertions.py@@ -5,9 +5,15 @@ from typing import Anyimport dspy-# compile time assertion-class Assert:+class DSPyAssertionError(AssertionError):+ """DSPy custom error message."""+ def __init__(self, msg: str, state: Any = None) -> None:+ super().__init__(msg)+ self.state = state++class Assert:+ """Compile time assertion."""def __init__(self, assert_fun, *args, **kwargs):self.assert_fun = assert_funself.args = args@@ -29,30 +35,8 @@ class Assert:# class RuntimeAssert(Assert): ...-class Assertions(dspy.Module):-- def __init__(self) -> None:- self.assertions = []-- def add(self, assertion: Assert):- self.assertions.append(assertion)-- def __call__(self, assertion: Assert):- self.add(assertion)- if self._compiled:- return True- else:- return assertion()---class DSPyAssertionError(AssertionError):-- def __init__(self, msg: str, state: Any = None) -> None:- super().__init__(msg)- self.state = state--def assert_transform(backtrack=2):+ """Decorator for transforming a function into a rewindable function."""def wrapper(func):def inner(*args, **kwargs):for i in range(backtrack):diff --git a/dspy/primitives/program.py b/dspy/primitives/program.pyindex d9278be44..886555858 100644--- a/dspy/primitives/program.py+++ b/dspy/primitives/program.py@@ -1,7 +1,7 @@import copyfrom dspy.primitives.module import BaseModule-from assertions import assert_transform+from dspy.primitives.assertions import assert_transformclass ProgramMeta(type):@@ -23,13 +23,13 @@ class Module(BaseModule, metaclass=ProgramMeta):def __init__(self):self._compiled = False- if getattr(self, "forward", False) and not getattr(- self.forward, "_decorated", False- ):- return setattr(self, "forward", assert_transform()(self.forward))def __call__(self, *args, **kwargs):- return self.forward(*args, **kwargs)+ if getattr(self, "forward", False) and not getattr(self.forward, "_decorated", False):+ wrapped_forward = assert_transform()(self.forward)+ return wrapped_forward(*args, **kwargs)+ else:+ return self.forward(*args, **kwargs)def named_predictors(self):from dspy.predict.predict import Predict