Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

feedback driven generation

Herumb Shandilyac26d5ba65e02
diff --git a/dspy/experimental/synthesizer/config.py b/dspy/experimental/synthesizer/config.py
index 609efbe12..640989c22 100644
--- a/dspy/experimental/synthesizer/config.py
+++ b/dspy/experimental/synthesizer/config.py
@@ -1,27 +1,22 @@
-from typing import Optional, Union
-
-from pydantic import BaseModel, field_validator
-
-import dspy
-
+from typing import Optional, Any
+from pydantic import BaseModel, model_validator
class SynthesizerArguments(BaseModel):
- # [TODO]
feedback_mode: Optional[str] = None
num_example_for_feedback: Optional[int] = None
- input_lm_model: Optional[dspy.LM] = None
- output_lm_model: Optional[dspy.LM] = None
- output_teacher_module: Optional[Union[dspy.Module, dspy.Predict]] = None
+ input_lm_model: Optional[Any] = None
+ output_lm_model: Optional[Any] = None
+ output_teacher_module: Optional[Any] = None
num_example_for_optim: Optional[int] = None
- @field_validator(fields=["feedback_mode", "num_example_for_feedback"])
- def validate_feedback_mode(cls, value):
- if value and value not in ["human", "llm"]:
+ @model_validator(mode='after')
+ def validate_feedback_mode(self):
+ if self.feedback_mode and self.feedback_mode not in ["human", "llm"]:
raise ValueError("Feedback mode should be either 'human' or 'llm'.")
- if value and not cls.num_example_for_feedback:
+ if self.feedback_mode and not self.num_example_for_feedback:
raise ValueError("Number of examples for feedback is required when feedback mode is provided.")
- return value
\ No newline at end of file
+ return self
\ No newline at end of file
diff --git a/dspy/experimental/synthesizer/instruction_suffixes.py b/dspy/experimental/synthesizer/instruction_suffixes.py
new file mode 100644
index 000000000..53404a2a6
--- /dev/null
+++ b/dspy/experimental/synthesizer/instruction_suffixes.py
@@ -0,0 +1,3 @@
+INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX = """\n\nI'll also be providing you some data I generated before hand, make sure the data you generate if consistent with task I provided but different from the data I provided in every way possible."""
+
+INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX = "\n\nAdditionally, I'll be providing you with feedback on the data you generate, while generating the data make sure to take into account the feedback I provide and try to improve the data you generate based on the feedback I provide."
\ No newline at end of file
diff --git a/dspy/experimental/synthesizer/instructions.py b/dspy/experimental/synthesizer/instructions.py
deleted file mode 100644
index 2fa65493a..000000000
--- a/dspy/experimental/synthesizer/instructions.py
+++ /dev/null
@@ -1,5 +0,0 @@
-INPUT_GENERATION_TASK_WITH_EXAMPLES = """Create synthetic data using the task description and the provided knowledge seed. Your task is to generate diverse and imaginative data that aligns with the given task description and knowledge seed. You are encouraged to be creative and not limit yourself, allowing for a wide range of synthetic data that reflects the characteristics and details provided in the task description. The data should be unique and varied, showcasing originality and creativity while maintaining relevance to the task and knowledge seed.
-
-Additionally I'll be providing you some data I generated before hand, make sure the data you generate if consistent with task I provided but different from the data I provided in every way possible."""
-
-INPUT_GENERATION_TASK_WITH_FEEDBACK = """"""
\ No newline at end of file
diff --git a/dspy/experimental/synthesizer/signatures.py b/dspy/experimental/synthesizer/signatures.py
index 0597dfdd5..444cbe12f 100644
--- a/dspy/experimental/synthesizer/signatures.py
+++ b/dspy/experimental/synthesizer/signatures.py
@@ -28,6 +28,22 @@ class ExplainTask(dspy.Signature):
desc="Explanation of the task.",
)
+class UpdateTaskDescriptionBasedOnFeedback(dspy.Signature):
+ """Update the task description based on the feedback provided. Ensure that the revised task description incorporates the feedback to improve its overall clarity and effectiveness. Focus on enhancing the task's goal and basic premise, without delving into specific data points, models, examples, algorithms, or technical intricacies. Your explanation should aim to clarify the task's fundamental objective and purpose."""
+
+ task_description = dspy.InputField(
+ prefix="Task Description:",
+ desc="Description of the task.",
+ )
+ feedback = dspy.InputField(
+ prefix="Feedback:",
+ desc="Feedback on the task description.",
+ )
+ updated_task_description = dspy.OutputField(
+ prefix="Task Description:",
+ desc="Updated description of the task.",
+ )
+
class GenerateFieldDescription(dspy.Signature):
"""Generate a concise and informative description for a given field based on the provided name and task description. This description should be no longer than 10 words and should be in simple english."""
diff --git a/dspy/experimental/synthesizer/synthesizer.py b/dspy/experimental/synthesizer/synthesizer.py
index d85f2cdb4..5930cfbbe 100644
--- a/dspy/experimental/synthesizer/synthesizer.py
+++ b/dspy/experimental/synthesizer/synthesizer.py
@@ -1,24 +1,31 @@
+import dspy
import random
-from collections.abc import Mapping
-from typing import List, Optional, Union
from datasets import Dataset
from tqdm import tqdm, trange
-
-import dspy
+from rich import print as rprint
+from collections.abc import Mapping
+from typing import List, Optional, Union
from .config import SynthesizerArguments
-from .instructions import INPUT_GENERATION_TASK_WITH_EXAMPLES
+from .instruction_suffixes import (
+ INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX,
+ INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX,
+)
from .signatures import (
ExplainTask,
GenerateFieldDescription,
GenerateInputFieldsData,
GenerateOutputFieldsData,
UnderstandTask,
+ UpdateTaskDescriptionBasedOnFeedback,
)
from .utils import format_examples
-__all__ = ["Synthesizer"]
+__all__ = [
+ "Synthesizer",
+ "SynthesizerArguments",
+]
class Synthesizer:
def __init__(self, config: SynthesizerArguments):
@@ -29,10 +36,33 @@ class Synthesizer:
self.explain_task = dspy.Predict(ExplainTask)
self.understand_task = dspy.Predict(UnderstandTask)
self.generate_field_description = dspy.Predict(GenerateFieldDescription)
+ self.update_task_description = dspy.Predict(UpdateTaskDescriptionBasedOnFeedback)
self.generate_input_data = GenerateInputFieldsData
self.generate_output_data = GenerateOutputFieldsData
+ def _gather_feedback(self, examples: dspy.Example) -> str:
+ if self.config.feedback_mode == "human":
+ input_keys = examples.inputs().keys()
+
+ print("-"*75)
+ print_text = "[bold blue]Generated Data:[bold blue]\n[bold red]Inputs:[bold red]\n"
+
+ for key in input_keys:
+ print_text += f"\t[bold yellow]{key}[bold yellow]: [green]{examples[key]}[green]\n"
+
+ rprint(print_text)
+ feedback = input("Provide feedback on the generated data: ")
+ print("-"*75)
+
+ return feedback
+
+ elif self.config.feedback_mode == "llm":
+ raise NotImplementedError("Feedback mode 'llm' is not implemented yet.")
+
+ else:
+ raise ValueError("Feedback mode should be either 'human' or 'llm'.")
+
def _get_field_data(self, key: str, keys_dict: Mapping[str, str]):
if key.startswith("$"):
field_details = self.generate_field_description(
@@ -137,7 +167,11 @@ class Synthesizer:
task_description, input_keys, output_keys = self._get_dataset_metadata(ground_source)
if self.config.num_example_for_optim:
- self.generate_input_data.__doc__ = INPUT_GENERATION_TASK_WITH_EXAMPLES
+ self.generate_input_data.__doc__ += INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX
+
+ if self.config.feedback_mode:
+ self.generate_input_data.__doc__ += INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX
+
self.generate_output_data.__doc__ = task_description
self.input_predictor, self.output_predictor = self._prepare_synthetic_data_predictors(
@@ -147,28 +181,23 @@ class Synthesizer:
)
data = []
+ feedback = ""
for idx in trange(0, num_data, batch_size, desc="Generating Synthetic Data"):
iter_temperature = 0.7+0.01*idx
iter_seed = random.randint(0, 1000000)
- inputs = None
+ kwargs = {
+ "task_description": task_description,
+ "knowledge_seed": iter_seed,
+ "config": dict(temperature=iter_temperature, n=batch_size),
+ }
+ if self.config.num_example_for_optim:
+ kwargs["ground_source"] = random.sample(ground_source, self.config.num_example_for_optim)
+
with dspy.context(lm=self.input_lm):
- if self.config.num_example_for_optim:
- example_for_optimization = random.sample(ground_source, self.config.num_example_for_optim)
- inputs = self.input_predictor(
- task_description=task_description,
- knowledge_seed=iter_seed,
- ground_source=example_for_optimization,
- config=dict(temperature=iter_temperature, n=batch_size),
- )
- else:
- inputs = self.input_predictor(
- task_description=task_description,
- knowledge_seed=iter_seed,
- config=dict(temperature=iter_temperature, n=batch_size),
- )
+ inputs = self.input_predictor(**kwargs)
input_kwargs = [{
key: getattr(completions, key)
@@ -191,6 +220,16 @@ class Synthesizer:
}
data.append(dspy.Example(**kwargs, **output_kwargs).with_inputs(*input_keys))
+
+ if self.config.feedback_mode and idx < self.config.num_example_for_feedback:
+ feedback = self._gather_feedback(data[-1])
+
+ task_description = self.update_task_description(
+ task_description=task_description,
+ feedback=feedback,
+ ).updated_task_description
+
+ self.output_predictor.signature.__doc__ = task_description
return data
diff --git a/dspy/experimental/synthesizer/utils.py b/dspy/experimental/synthesizer/utils.py
index f08b142e1..b52ae9016 100644
--- a/dspy/experimental/synthesizer/utils.py
+++ b/dspy/experimental/synthesizer/utils.py
@@ -1,7 +1,5 @@
-from typing import List
-
import dspy
-
+from typing import List
def format_examples(examples: List[dspy.Example]) -> str:
if isinstance(examples, str):
diff --git a/poetry.lock b/poetry.lock
index 30ec76981..ebf68afe9 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2021,7 +2021,7 @@ testing = ["coverage", "pyyaml"]
name = "markdown-it-py"
version = "2.2.0"
description = "Python port of markdown-it. Markdown parsing, done right!"
-optional = true
+optional = false
python-versions = ">=3.7"
files = [
{file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"},
@@ -2163,7 +2163,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
name = "mdurl"
version = "0.1.2"
description = "Markdown URL utilities"
-optional = true
+optional = false
python-versions = ">=3.7"
files = [
{file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
@@ -3703,7 +3703,6 @@ files = [
{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
- {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -4036,6 +4035,24 @@ requests = ">=2.0.0"
[package.extras]
rsa = ["oauthlib[signedtoken] (>=3.0.0)"]
+[[package]]
+name = "rich"
+version = "13.7.1"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+ {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"},
+ {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"},
+]
+
+[package.dependencies]
+markdown-it-py = ">=2.2.0"
+pygments = ">=2.13.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<9)"]
+
[[package]]
name = "rpds-py"
version = "0.18.0"
@@ -5568,4 +5585,4 @@ weaviate = ["weaviate-client"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.9,<3.12"
-content-hash = "8b4cc583653becb3be9f5bc4c34cdf5ced146ba32157a5bb4bdc7885291c0403"
+content-hash = "ccfd76e32c6c94d32fd3f81815b5d499d4ba9b6f4707bc547972a2c1e0846745"
diff --git a/pyproject.toml b/pyproject.toml
index e2c719eaf..749b15f6b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -101,6 +101,7 @@ sphinx_rtd_theme = { version = "*", optional = true }
autodoc_pydantic = { version = "*", optional = true }
sphinx-reredirects = { version = "^0.1.2", optional = true }
sphinx-automodapi = { version = "0.16.0", optional = true }
+rich = "^13.7.1"
[tool.poetry.group.test.dependencies]