Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Assert examples with real assertions

Shangyint345c4ef51895
diff --git a/assert_example.py b/assert_example.py
index d9b6cc9b0..bfe6a6092 100644
--- a/assert_example.py
+++ b/assert_example.py
@@ -6,15 +6,19 @@ from dspy.datasets import HotPotQA
from dspy.teleprompt import BootstrapFewShot
# pipeline configs
-turbo = dspy.OpenAI(model='gpt-3.5-turbo')
-colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')
+turbo = dspy.OpenAI(model="gpt-3.5-turbo")
+colbertv2_wiki17_abstracts = dspy.ColBERTv2(
+ url="http://20.102.90.50:2017/wiki17_abstracts"
+)
dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)
# load dataset
-dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
-trainset = [x.with_inputs('question') for x in dataset.train]
-devset = [x.with_inputs('question') for x in dataset.dev]
+dataset = HotPotQA(
+ train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0
+)
+trainset = [x.with_inputs("question") for x in dataset.train]
+devset = [x.with_inputs("question") for x in dataset.dev]
# signatures of dspy modules
@@ -24,7 +28,8 @@ class GenerateAnswer(dspy.Signature):
context = dspy.InputField(desc="may contain relevant facts")
question = dspy.InputField()
answer = dspy.OutputField(desc="often between 1 and 5 words")
-
+
+
class GenerateSearchQuery(dspy.Signature):
"""Write a simple search query that will help answer a complex question."""
@@ -32,20 +37,26 @@ class GenerateSearchQuery(dspy.Signature):
question = dspy.InputField()
query = dspy.OutputField()
+
# validation logic for verifying traces
def validate_context_and_answer_and_hops(example, pred, trace=None):
if not dspy.evaluate.answer_exact_match(example, pred):
return False
-
+
if not dspy.evaluate.answer_passage_match(example, pred):
return False
- hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]
+ hops = [example.question] + [
+ outputs.query for *_, outputs in trace if "query" in outputs
+ ]
if max([len(h) for h in hops]) > 100:
return False
-
- if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): # 4
+
+ if any(
+ dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)
+ for idx in range(2, len(hops))
+ ): # 4
return False
return True
@@ -54,26 +65,70 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
########################## NEW STUFF ##########################
+def validate_query_distinction_local(previous_queries, query):
+ if previous_queries == []:
+ return True
+ if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):
+ return False
+ return True
+
+
+# class GenerateSearchQuery(dspy.Signature):
+# query = dspy.InputField(desc="current query")
+# previous_queries = dspy.InputField(desc="previous queries")
+# duplication = dspy.OutputField(
+# desc="Whether the current query is similar to previous queries, only return True or False"
+# )
+
+
+# class QueryDistinction(dspy.Module):
+
+# def __init__(self):
+# super().__init__()
+# self.generate_query = dspy.Predict(GenerateSearchQuery)
+
+# def forward(self, query, previous_queries):
+# duplication = self.generate_query(
+# query=query, previous_queries=previous_queries
+# ).duplication
+# return duplication
+
+
+# def validate_query_distinction_LM(previous_queries, query):
+# query_distinction = QueryDistinction()
+# duplication = query_distinction(query=query, previous_queries=previous_queries)
+# return True if duplication == "False" else False
+
+
# declaration of dspy program
class SimplifiedBaleen(dspy.Module):
+
def __init__(self, passages_per_hop=2, max_hops=2):
super().__init__()
- self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]
+ self.generate_query = [
+ dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)
+ ]
self.retrieve = dspy.Retrieve(k=passages_per_hop)
self.generate_answer = dspy.ChainOfThought(GenerateAnswer)
self.max_hops = max_hops
-
+
def forward(self, question):
+ print("question is ", question)
context = []
-
+ previous_queries = []
for hop in range(self.max_hops):
query = self.generate_query[hop](context=context, question=question).query
- # assertion
- print(f"{hop}. Query length @ {id(self.generate_query[hop])} w/ config {self.generate_query[hop].get_config()}: {len(query)}")
- dspy.Assert(lambda x: len(x) <= 100, query, msg="Query length should be less than 100 characters.")
-
+ # turbo.inspect_history(n=1)
+ print("query is ", query)
+ dspy.Assert(
+ validate_query_distinction_local,
+ previous_queries,
+ query,
+ msg=f"Queries should be different with previous queries {previous_queries}",
+ )
+ previous_queries.append(query)
passages = self.retrieve(query).passages
context = deduplicate(context + passages)
@@ -81,6 +136,7 @@ class SimplifiedBaleen(dspy.Module):
return dspy.Prediction(context=context, answer=pred.answer)
+
# compile dspy program using a teleprompter (optimizer)
teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)
@@ -90,4 +146,4 @@ compiled_baleen = teleprompter.compile(student, teacher=teacher, trainset=trains
my_question = "How many storeys are in the castle that David Gregory inherited?"
pred = compiled_baleen(my_question)
-turbo.inspect_history(n=3)
\ No newline at end of file
+turbo.inspect_history(n=3)
diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.py
index 28bb9a88e..ee95a8651 100644
--- a/dspy/primitives/assertions.py
+++ b/dspy/primitives/assertions.py
@@ -11,7 +11,7 @@ class DSPyAssertionError(AssertionError):
def __init__(self, msg: str, state: Any = None) -> None:
super().__init__(msg)
- self.msg = msg
+ self.msg = f"Assertion Error: {msg}"
self.state = state