Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

adding fixes and refactoring

manishshettym04ae621ca5cd
diff --git a/assert_example.py b/assert_example.py
new file mode 100644
index 000000000..bce2b5621
--- /dev/null
+++ b/assert_example.py
@@ -0,0 +1,92 @@
+import os
+
+import dspy
+from dsp.utils import deduplicate
+from dspy.datasets import HotPotQA
+from dspy.teleprompt import BootstrapFewShot
+
+# pipeline configs
+turbo = dspy.OpenAI(model='gpt-3.5-turbo')
+colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')
+dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
+
+
+# load dataset
+dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
+trainset = [x.with_inputs('question') for x in dataset.train]
+devset = [x.with_inputs('question') for x in dataset.dev]
+
+
+# signatures of dspy modules
+class GenerateAnswer(dspy.Signature):
+ """Answer questions with short factoid answers."""
+
+ context = dspy.InputField(desc="may contain relevant facts")
+ question = dspy.InputField()
+ answer = dspy.OutputField(desc="often between 1 and 5 words")
+
+class GenerateSearchQuery(dspy.Signature):
+ """Write a simple search query that will help answer a complex question."""
+
+ context = dspy.InputField(desc="may contain relevant facts")
+ question = dspy.InputField()
+ query = dspy.OutputField()
+
+# validation logic for verifying traces
+def validate_context_and_answer_and_hops(example, pred, trace=None):
+ if not dspy.evaluate.answer_exact_match(example, pred):
+ return False
+
+ if not dspy.evaluate.answer_passage_match(example, pred):
+ return False
+
+ hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]
+
+ if max([len(h) for h in hops]) > 100:
+ return False
+
+ if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): # 4
+ return False
+
+ return True
+
+
+########################## NEW STUFF ##########################
+
+
+# declaration of dspy program
+class SimplifiedBaleen(dspy.Module):
+ def __init__(self, passages_per_hop=2, max_hops=2):
+ super().__init__()
+
+ self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]
+ self.retrieve = dspy.Retrieve(k=passages_per_hop)
+ self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
+ self.max_hops = max_hops
+
+ def forward(self, question):
+ context = []
+
+ for hop in range(self.max_hops):
+ query = self.generate_query[hop](context=context, question=question).query
+
+ # assertion
+ dspy.Assert(lambda x: len(x) <= 100, query)
+
+ passages = self.retrieve(query).passages
+ context = deduplicate(context + passages)
+
+ pred = self.generate_answer(context=context, question=question)
+
+ return dspy.Prediction(context=context, answer=pred.answer)
+
+# compile dspy program using a teleprompter (optimizer)
+teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
+
+student = SimplifiedBaleen()
+teacher = SimplifiedBaleen(passages_per_hop=2)
+compiled_baleen = teleprompter.compile(student, teacher=teacher, trainset=trainset)
+
+my_question = "How many storeys are in the castle that David Gregory inherited?"
+pred = compiled_baleen(my_question)
+turbo.inspect_history(n=3)
\ No newline at end of file
diff --git a/dspy/__init__.py b/dspy/__init__.py
index 26ea97644..18b70008d 100644
--- a/dspy/__init__.py
+++ b/dspy/__init__.py
@@ -4,7 +4,6 @@ from .signatures import *
from .retrieve import *
from .predict import *
from .primitives import *
-from .assertions import *
# from .evaluation import *
diff --git a/dspy/assertions/__init__.py b/dspy/assertions/__init__.py
deleted file mode 100644
index 6da1aa37e..000000000
--- a/dspy/assertions/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .assertions import *
diff --git a/dspy/primitives/__init__.py b/dspy/primitives/__init__.py
index f99f3af2d..b61bf55e1 100644
--- a/dspy/primitives/__init__.py
+++ b/dspy/primitives/__init__.py
@@ -1,3 +1,4 @@
from .example import *
from .program import *
-from .prediction import *
\ No newline at end of file
+from .prediction import *
+from .assertions import *
\ No newline at end of file
diff --git a/dspy/assertions/assertions.py b/dspy/primitives/assertions.py
similarity index 79%
rename from dspy/assertions/assertions.py
rename to dspy/primitives/assertions.py
index ef2f04143..5f1376a29 100644
--- a/dspy/assertions/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -5,9 +5,15 @@ from typing import Any
import dspy
-# compile time assertion
-class Assert:
+class DSPyAssertionError(AssertionError):
+ """DSPy custom error message."""
+ def __init__(self, msg: str, state: Any = None) -> None:
+ super().__init__(msg)
+ self.state = state
+
+class Assert:
+ """Compile time assertion."""
def __init__(self, assert_fun, *args, **kwargs):
self.assert_fun = assert_fun
self.args = args
@@ -29,30 +35,8 @@ class Assert:
# class RuntimeAssert(Assert): ...
-class Assertions(dspy.Module):
-
- def __init__(self) -> None:
- self.assertions = []
-
- def add(self, assertion: Assert):
- self.assertions.append(assertion)
-
- def __call__(self, assertion: Assert):
- self.add(assertion)
- if self._compiled:
- return True
- else:
- return assertion()
-
-
-class DSPyAssertionError(AssertionError):
-
- def __init__(self, msg: str, state: Any = None) -> None:
- super().__init__(msg)
- self.state = state
-
-
def assert_transform(backtrack=2):
+ """Decorator for transforming a function into a rewindable function."""
def wrapper(func):
def inner(*args, **kwargs):
for i in range(backtrack):
diff --git a/dspy/primitives/program.py b/dspy/primitives/program.py
index d9278be44..886555858 100644
--- a/dspy/primitives/program.py
+++ b/dspy/primitives/program.py
@@ -1,7 +1,7 @@
import copy
from dspy.primitives.module import BaseModule
-from assertions import assert_transform
+from dspy.primitives.assertions import assert_transform
class ProgramMeta(type):
@@ -23,13 +23,13 @@ class Module(BaseModule, metaclass=ProgramMeta):
def __init__(self):
self._compiled = False
- if getattr(self, "forward", False) and not getattr(
- self.forward, "_decorated", False
- ):
- return setattr(self, "forward", assert_transform()(self.forward))
def __call__(self, *args, **kwargs):
- return self.forward(*args, **kwargs)
+ if getattr(self, "forward", False) and not getattr(self.forward, "_decorated", False):
+ wrapped_forward = assert_transform()(self.forward)
+ return wrapped_forward(*args, **kwargs)
+ else:
+ return self.forward(*args, **kwargs)
def named_predictors(self):
from dspy.predict.predict import Predict