Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Automated feedback incorporations

Herumb Shandilya04bedffccf8e
diff --git a/dspy/experimental/synthesizer/signatures.py b/dspy/experimental/synthesizer/signatures.py
index 444cbe12f..e1a50c689 100644
--- a/dspy/experimental/synthesizer/signatures.py
+++ b/dspy/experimental/synthesizer/signatures.py
@@ -44,6 +44,23 @@ class UpdateTaskDescriptionBasedOnFeedback(dspy.Signature):
desc="Updated description of the task.",
)
+class GetFeedbackOnGeneration(dspy.Signature):
+ """Provide constructive feedback on the synthetic data generated, focusing on its quality, relevance, and diversity. Highlight any areas that require improvement and offer suggestions for enhancement. The feedback should center on the overall effectiveness of the synthetic data in aligning with the task description and knowledge seed. Avoid delving into specific data points, models, examples, algorithms, or technical intricacies. Your feedback should be critical but constructive, aiming to improve the synthetic data and the task description."""
+
+ synthetic_data = dspy.InputField(
+ prefix="Synthetic Data:",
+ desc="Synthetic data generated.",
+ format=format_examples,
+ )
+ task_description = dspy.InputField(
+ prefix="Task Description:",
+ desc="Description of the task the synthetic data is aligned with.",
+ )
+ feedback = dspy.OutputField(
+ prefix="Feedback:",
+ desc="Feedback on the synthetic data.",
+ )
+
class GenerateFieldDescription(dspy.Signature):
"""Generate a concise and informative description for a given field based on the provided name and task description. This description should be no longer than 10 words and should be in simple english."""
diff --git a/dspy/experimental/synthesizer/synthesizer.py b/dspy/experimental/synthesizer/synthesizer.py
index 5930cfbbe..a3ffdcda8 100644
--- a/dspy/experimental/synthesizer/synthesizer.py
+++ b/dspy/experimental/synthesizer/synthesizer.py
@@ -17,6 +17,7 @@ from .signatures import (
GenerateFieldDescription,
GenerateInputFieldsData,
GenerateOutputFieldsData,
+ GetFeedbackOnGeneration,
UnderstandTask,
UpdateTaskDescriptionBasedOnFeedback,
)
@@ -35,6 +36,7 @@ class Synthesizer:
self.explain_task = dspy.Predict(ExplainTask)
self.understand_task = dspy.Predict(UnderstandTask)
+ self.get_feedback_on_generation = dspy.Predict(GetFeedbackOnGeneration)
self.generate_field_description = dspy.Predict(GenerateFieldDescription)
self.update_task_description = dspy.Predict(UpdateTaskDescriptionBasedOnFeedback)
@@ -58,7 +60,14 @@ class Synthesizer:
return feedback
elif self.config.feedback_mode == "llm":
- raise NotImplementedError("Feedback mode 'llm' is not implemented yet.")
+ feedback = self.get_feedback_on_generation(
+ synthetic_data=[examples],
+ task_description=self.generate_output_data.__doc__,
+ )
+
+ print(feedback.feedback)
+
+ return feedback.feedback
else:
raise ValueError("Feedback mode should be either 'human' or 'llm'.")