DSPy: The framework for programming—not prompting—language models
Python
Automated feedback incorporations
Herumb Shandilya04bedffccf8e
diff --git a/dspy/experimental/synthesizer/signatures.py b/dspy/experimental/synthesizer/signatures.pyindex 444cbe12f..e1a50c689 100644--- a/dspy/experimental/synthesizer/signatures.py+++ b/dspy/experimental/synthesizer/signatures.py@@ -44,6 +44,23 @@ class UpdateTaskDescriptionBasedOnFeedback(dspy.Signature):desc="Updated description of the task.",)+class GetFeedbackOnGeneration(dspy.Signature):+ """Provide constructive feedback on the synthetic data generated, focusing on its quality, relevance, and diversity. Highlight any areas that require improvement and offer suggestions for enhancement. The feedback should center on the overall effectiveness of the synthetic data in aligning with the task description and knowledge seed. Avoid delving into specific data points, models, examples, algorithms, or technical intricacies. Your feedback should be critical but constructive, aiming to improve the synthetic data and the task description."""++ synthetic_data = dspy.InputField(+ prefix="Synthetic Data:",+ desc="Synthetic data generated.",+ format=format_examples,+ )+ task_description = dspy.InputField(+ prefix="Task Description:",+ desc="Description of the task the synthetic data is aligned with.",+ )+ feedback = dspy.OutputField(+ prefix="Feedback:",+ desc="Feedback on the synthetic data.",+ )+class GenerateFieldDescription(dspy.Signature):"""Generate a concise and informative description for a given field based on the provided name and task description. This description should be no longer than 10 words and should be in simple english."""diff --git a/dspy/experimental/synthesizer/synthesizer.py b/dspy/experimental/synthesizer/synthesizer.pyindex 5930cfbbe..a3ffdcda8 100644--- a/dspy/experimental/synthesizer/synthesizer.py+++ b/dspy/experimental/synthesizer/synthesizer.py@@ -17,6 +17,7 @@ from .signatures import (GenerateFieldDescription,GenerateInputFieldsData,GenerateOutputFieldsData,+ GetFeedbackOnGeneration,UnderstandTask,UpdateTaskDescriptionBasedOnFeedback,)@@ -35,6 +36,7 @@ class Synthesizer:self.explain_task = dspy.Predict(ExplainTask)self.understand_task = dspy.Predict(UnderstandTask)+ self.get_feedback_on_generation = dspy.Predict(GetFeedbackOnGeneration)self.generate_field_description = dspy.Predict(GenerateFieldDescription)self.update_task_description = dspy.Predict(UpdateTaskDescriptionBasedOnFeedback)@@ -58,7 +60,14 @@ class Synthesizer:return feedbackelif self.config.feedback_mode == "llm":- raise NotImplementedError("Feedback mode 'llm' is not implemented yet.")+ feedback = self.get_feedback_on_generation(+ synthetic_data=[examples],+ task_description=self.generate_output_data.__doc__,+ )++ print(feedback.feedback)++ return feedback.feedbackelse:raise ValueError("Feedback mode should be either 'human' or 'llm'.")