DSPy: The framework for programming—not prompting—language models
Python
Assert examples with real assertions
Shangyint345c4ef51895
diff --git a/assert_example.py b/assert_example.pyindex d9b6cc9b0..bfe6a6092 100644--- a/assert_example.py+++ b/assert_example.py@@ -6,15 +6,19 @@ from dspy.datasets import HotPotQAfrom dspy.teleprompt import BootstrapFewShot# pipeline configs-turbo = dspy.OpenAI(model='gpt-3.5-turbo')-colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')+turbo = dspy.OpenAI(model="gpt-3.5-turbo")+colbertv2_wiki17_abstracts = dspy.ColBERTv2(+ url="http://20.102.90.50:2017/wiki17_abstracts"+)dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts)# load dataset-dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)-trainset = [x.with_inputs('question') for x in dataset.train]-devset = [x.with_inputs('question') for x in dataset.dev]+dataset = HotPotQA(+ train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0+)+trainset = [x.with_inputs("question") for x in dataset.train]+devset = [x.with_inputs("question") for x in dataset.dev]# signatures of dspy modules@@ -24,7 +28,8 @@ class GenerateAnswer(dspy.Signature):context = dspy.InputField(desc="may contain relevant facts")question = dspy.InputField()answer = dspy.OutputField(desc="often between 1 and 5 words")-++class GenerateSearchQuery(dspy.Signature):"""Write a simple search query that will help answer a complex question."""@@ -32,20 +37,26 @@ class GenerateSearchQuery(dspy.Signature):question = dspy.InputField()query = dspy.OutputField()+# validation logic for verifying tracesdef validate_context_and_answer_and_hops(example, pred, trace=None):if not dspy.evaluate.answer_exact_match(example, pred):return False-+if not dspy.evaluate.answer_passage_match(example, pred):return False- hops = [example.question] + [outputs.query for *_, outputs in trace if 'query' in outputs]+ hops = [example.question] + [+ outputs.query for *_, outputs in trace if "query" in outputs+ ]if max([len(h) for h in hops]) > 100:return False-- if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))): # 4++ if any(+ dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)+ for idx in range(2, len(hops))+ ): # 4return Falsereturn True@@ -54,26 +65,70 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):########################## NEW STUFF ##########################+def validate_query_distinction_local(previous_queries, query):+ if previous_queries == []:+ return True+ if dspy.evaluate.answer_exact_match_str(query, previous_queries, frac=0.8):+ return False+ return True+++# class GenerateSearchQuery(dspy.Signature):+# query = dspy.InputField(desc="current query")+# previous_queries = dspy.InputField(desc="previous queries")+# duplication = dspy.OutputField(+# desc="Whether the current query is similar to previous queries, only return True or False"+# )+++# class QueryDistinction(dspy.Module):++# def __init__(self):+# super().__init__()+# self.generate_query = dspy.Predict(GenerateSearchQuery)++# def forward(self, query, previous_queries):+# duplication = self.generate_query(+# query=query, previous_queries=previous_queries+# ).duplication+# return duplication+++# def validate_query_distinction_LM(previous_queries, query):+# query_distinction = QueryDistinction()+# duplication = query_distinction(query=query, previous_queries=previous_queries)+# return True if duplication == "False" else False++# declaration of dspy programclass SimplifiedBaleen(dspy.Module):+def __init__(self, passages_per_hop=2, max_hops=2):super().__init__()- self.generate_query = [dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)]+ self.generate_query = [+ dspy.ChainOfThought(GenerateSearchQuery) for _ in range(max_hops)+ ]self.retrieve = dspy.Retrieve(k=passages_per_hop)self.generate_answer = dspy.ChainOfThought(GenerateAnswer)self.max_hops = max_hops-+def forward(self, question):+ print("question is ", question)context = []-+ previous_queries = []for hop in range(self.max_hops):query = self.generate_query[hop](context=context, question=question).query- # assertion- print(f"{hop}. Query length @ {id(self.generate_query[hop])} w/ config {self.generate_query[hop].get_config()}: {len(query)}")- dspy.Assert(lambda x: len(x) <= 100, query, msg="Query length should be less than 100 characters.")-+ # turbo.inspect_history(n=1)+ print("query is ", query)+ dspy.Assert(+ validate_query_distinction_local,+ previous_queries,+ query,+ msg=f"Queries should be different with previous queries {previous_queries}",+ )+ previous_queries.append(query)passages = self.retrieve(query).passagescontext = deduplicate(context + passages)@@ -81,6 +136,7 @@ class SimplifiedBaleen(dspy.Module):return dspy.Prediction(context=context, answer=pred.answer)+# compile dspy program using a teleprompter (optimizer)teleprompter = BootstrapFewShot(metric=validate_context_and_answer_and_hops)@@ -90,4 +146,4 @@ compiled_baleen = teleprompter.compile(student, teacher=teacher, trainset=trainsmy_question = "How many storeys are in the castle that David Gregory inherited?"pred = compiled_baleen(my_question)-turbo.inspect_history(n=3)\ No newline at end of file+turbo.inspect_history(n=3)diff --git a/dspy/primitives/assertions.py b/dspy/primitives/assertions.pyindex 28bb9a88e..ee95a8651 100644--- a/dspy/primitives/assertions.py+++ b/dspy/primitives/assertions.py@@ -11,7 +11,7 @@ class DSPyAssertionError(AssertionError):def __init__(self, msg: str, state: Any = None) -> None:super().__init__(msg)- self.msg = msg+ self.msg = f"Assertion Error: {msg}"self.state = state