DSPy: The framework for programming—not prompting—language models
Python
feedback driven generation
Herumb Shandilyac26d5ba65e02
diff --git a/dspy/experimental/synthesizer/config.py b/dspy/experimental/synthesizer/config.pyindex 609efbe12..640989c22 100644--- a/dspy/experimental/synthesizer/config.py+++ b/dspy/experimental/synthesizer/config.py@@ -1,27 +1,22 @@-from typing import Optional, Union--from pydantic import BaseModel, field_validator--import dspy-+from typing import Optional, Any+from pydantic import BaseModel, model_validatorclass SynthesizerArguments(BaseModel):- # [TODO]feedback_mode: Optional[str] = Nonenum_example_for_feedback: Optional[int] = None- input_lm_model: Optional[dspy.LM] = None- output_lm_model: Optional[dspy.LM] = None- output_teacher_module: Optional[Union[dspy.Module, dspy.Predict]] = None+ input_lm_model: Optional[Any] = None+ output_lm_model: Optional[Any] = None+ output_teacher_module: Optional[Any] = Nonenum_example_for_optim: Optional[int] = None- @field_validator(fields=["feedback_mode", "num_example_for_feedback"])- def validate_feedback_mode(cls, value):- if value and value not in ["human", "llm"]:+ @model_validator(mode='after')+ def validate_feedback_mode(self):+ if self.feedback_mode and self.feedback_mode not in ["human", "llm"]:raise ValueError("Feedback mode should be either 'human' or 'llm'.")- if value and not cls.num_example_for_feedback:+ if self.feedback_mode and not self.num_example_for_feedback:raise ValueError("Number of examples for feedback is required when feedback mode is provided.")- return value\ No newline at end of file+ return self\ No newline at end of filediff --git a/dspy/experimental/synthesizer/instruction_suffixes.py b/dspy/experimental/synthesizer/instruction_suffixes.pynew file mode 100644index 000000000..53404a2a6--- /dev/null+++ b/dspy/experimental/synthesizer/instruction_suffixes.py@@ -0,0 +1,3 @@+INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX = """\n\nI'll also be providing you some data I generated before hand, make sure the data you generate if consistent with task I provided but different from the data I provided in every way possible."""++INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX = "\n\nAdditionally, I'll be providing you with feedback on the data you generate, while generating the data make sure to take into account the feedback I provide and try to improve the data you generate based on the feedback I provide."\ No newline at end of filediff --git a/dspy/experimental/synthesizer/instructions.py b/dspy/experimental/synthesizer/instructions.pydeleted file mode 100644index 2fa65493a..000000000--- a/dspy/experimental/synthesizer/instructions.py+++ /dev/null@@ -1,5 +0,0 @@-INPUT_GENERATION_TASK_WITH_EXAMPLES = """Create synthetic data using the task description and the provided knowledge seed. Your task is to generate diverse and imaginative data that aligns with the given task description and knowledge seed. You are encouraged to be creative and not limit yourself, allowing for a wide range of synthetic data that reflects the characteristics and details provided in the task description. The data should be unique and varied, showcasing originality and creativity while maintaining relevance to the task and knowledge seed.--Additionally I'll be providing you some data I generated before hand, make sure the data you generate if consistent with task I provided but different from the data I provided in every way possible."""--INPUT_GENERATION_TASK_WITH_FEEDBACK = """"""\ No newline at end of filediff --git a/dspy/experimental/synthesizer/signatures.py b/dspy/experimental/synthesizer/signatures.pyindex 0597dfdd5..444cbe12f 100644--- a/dspy/experimental/synthesizer/signatures.py+++ b/dspy/experimental/synthesizer/signatures.py@@ -28,6 +28,22 @@ class ExplainTask(dspy.Signature):desc="Explanation of the task.",)+class UpdateTaskDescriptionBasedOnFeedback(dspy.Signature):+ """Update the task description based on the feedback provided. Ensure that the revised task description incorporates the feedback to improve its overall clarity and effectiveness. Focus on enhancing the task's goal and basic premise, without delving into specific data points, models, examples, algorithms, or technical intricacies. Your explanation should aim to clarify the task's fundamental objective and purpose."""++ task_description = dspy.InputField(+ prefix="Task Description:",+ desc="Description of the task.",+ )+ feedback = dspy.InputField(+ prefix="Feedback:",+ desc="Feedback on the task description.",+ )+ updated_task_description = dspy.OutputField(+ prefix="Task Description:",+ desc="Updated description of the task.",+ )+class GenerateFieldDescription(dspy.Signature):"""Generate a concise and informative description for a given field based on the provided name and task description. This description should be no longer than 10 words and should be in simple english."""diff --git a/dspy/experimental/synthesizer/synthesizer.py b/dspy/experimental/synthesizer/synthesizer.pyindex d85f2cdb4..5930cfbbe 100644--- a/dspy/experimental/synthesizer/synthesizer.py+++ b/dspy/experimental/synthesizer/synthesizer.py@@ -1,24 +1,31 @@+import dspyimport random-from collections.abc import Mapping-from typing import List, Optional, Unionfrom datasets import Datasetfrom tqdm import tqdm, trange--import dspy+from rich import print as rprint+from collections.abc import Mapping+from typing import List, Optional, Unionfrom .config import SynthesizerArguments-from .instructions import INPUT_GENERATION_TASK_WITH_EXAMPLES+from .instruction_suffixes import (+ INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX,+ INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX,+)from .signatures import (ExplainTask,GenerateFieldDescription,GenerateInputFieldsData,GenerateOutputFieldsData,UnderstandTask,+ UpdateTaskDescriptionBasedOnFeedback,)from .utils import format_examples-__all__ = ["Synthesizer"]+__all__ = [+ "Synthesizer",+ "SynthesizerArguments",+]class Synthesizer:def __init__(self, config: SynthesizerArguments):@@ -29,10 +36,33 @@ class Synthesizer:self.explain_task = dspy.Predict(ExplainTask)self.understand_task = dspy.Predict(UnderstandTask)self.generate_field_description = dspy.Predict(GenerateFieldDescription)+ self.update_task_description = dspy.Predict(UpdateTaskDescriptionBasedOnFeedback)self.generate_input_data = GenerateInputFieldsDataself.generate_output_data = GenerateOutputFieldsData+ def _gather_feedback(self, examples: dspy.Example) -> str:+ if self.config.feedback_mode == "human":+ input_keys = examples.inputs().keys()++ print("-"*75)+ print_text = "[bold blue]Generated Data:[bold blue]\n[bold red]Inputs:[bold red]\n"++ for key in input_keys:+ print_text += f"\t[bold yellow]{key}[bold yellow]: [green]{examples[key]}[green]\n"++ rprint(print_text)+ feedback = input("Provide feedback on the generated data: ")+ print("-"*75)++ return feedback++ elif self.config.feedback_mode == "llm":+ raise NotImplementedError("Feedback mode 'llm' is not implemented yet.")++ else:+ raise ValueError("Feedback mode should be either 'human' or 'llm'.")+def _get_field_data(self, key: str, keys_dict: Mapping[str, str]):if key.startswith("$"):field_details = self.generate_field_description(@@ -137,7 +167,11 @@ class Synthesizer:task_description, input_keys, output_keys = self._get_dataset_metadata(ground_source)if self.config.num_example_for_optim:- self.generate_input_data.__doc__ = INPUT_GENERATION_TASK_WITH_EXAMPLES+ self.generate_input_data.__doc__ += INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX++ if self.config.feedback_mode:+ self.generate_input_data.__doc__ += INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX+self.generate_output_data.__doc__ = task_descriptionself.input_predictor, self.output_predictor = self._prepare_synthetic_data_predictors(@@ -147,28 +181,23 @@ class Synthesizer:)data = []+ feedback = ""for idx in trange(0, num_data, batch_size, desc="Generating Synthetic Data"):iter_temperature = 0.7+0.01*idxiter_seed = random.randint(0, 1000000)- inputs = None+ kwargs = {+ "task_description": task_description,+ "knowledge_seed": iter_seed,+ "config": dict(temperature=iter_temperature, n=batch_size),+ }+ if self.config.num_example_for_optim:+ kwargs["ground_source"] = random.sample(ground_source, self.config.num_example_for_optim)+with dspy.context(lm=self.input_lm):- if self.config.num_example_for_optim:- example_for_optimization = random.sample(ground_source, self.config.num_example_for_optim)- inputs = self.input_predictor(- task_description=task_description,- knowledge_seed=iter_seed,- ground_source=example_for_optimization,- config=dict(temperature=iter_temperature, n=batch_size),- )- else:- inputs = self.input_predictor(- task_description=task_description,- knowledge_seed=iter_seed,- config=dict(temperature=iter_temperature, n=batch_size),- )+ inputs = self.input_predictor(**kwargs)input_kwargs = [{key: getattr(completions, key)@@ -191,6 +220,16 @@ class Synthesizer:}data.append(dspy.Example(**kwargs, **output_kwargs).with_inputs(*input_keys))++ if self.config.feedback_mode and idx < self.config.num_example_for_feedback:+ feedback = self._gather_feedback(data[-1])++ task_description = self.update_task_description(+ task_description=task_description,+ feedback=feedback,+ ).updated_task_description++ self.output_predictor.signature.__doc__ = task_descriptionreturn datadiff --git a/dspy/experimental/synthesizer/utils.py b/dspy/experimental/synthesizer/utils.pyindex f08b142e1..b52ae9016 100644--- a/dspy/experimental/synthesizer/utils.py+++ b/dspy/experimental/synthesizer/utils.py@@ -1,7 +1,5 @@-from typing import List-import dspy-+from typing import Listdef format_examples(examples: List[dspy.Example]) -> str:if isinstance(examples, str):diff --git a/poetry.lock b/poetry.lockindex 30ec76981..ebf68afe9 100644--- a/poetry.lock+++ b/poetry.lock@@ -2021,7 +2021,7 @@ testing = ["coverage", "pyyaml"]name = "markdown-it-py"version = "2.2.0"description = "Python port of markdown-it. Markdown parsing, done right!"-optional = true+optional = falsepython-versions = ">=3.7"files = [{file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"},@@ -2163,7 +2163,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]name = "mdurl"version = "0.1.2"description = "Markdown URL utilities"-optional = true+optional = falsepython-versions = ">=3.7"files = [{file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},@@ -3703,7 +3703,6 @@ files = [{file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},{file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},{file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},- {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},{file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},{file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},{file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},@@ -4036,6 +4035,24 @@ requests = ">=2.0.0"[package.extras]rsa = ["oauthlib[signedtoken] (>=3.0.0)"]+[[package]]+name = "rich"+version = "13.7.1"+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"+optional = false+python-versions = ">=3.7.0"+files = [+ {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"},+ {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"},+]++[package.dependencies]+markdown-it-py = ">=2.2.0"+pygments = ">=2.13.0,<3.0.0"++[package.extras]+jupyter = ["ipywidgets (>=7.5.1,<9)"]+[[package]]name = "rpds-py"version = "0.18.0"@@ -5568,4 +5585,4 @@ weaviate = ["weaviate-client"][metadata]lock-version = "2.0"python-versions = ">=3.9,<3.12"-content-hash = "8b4cc583653becb3be9f5bc4c34cdf5ced146ba32157a5bb4bdc7885291c0403"+content-hash = "ccfd76e32c6c94d32fd3f81815b5d499d4ba9b6f4707bc547972a2c1e0846745"diff --git a/pyproject.toml b/pyproject.tomlindex e2c719eaf..749b15f6b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -101,6 +101,7 @@ sphinx_rtd_theme = { version = "*", optional = true }autodoc_pydantic = { version = "*", optional = true }sphinx-reredirects = { version = "^0.1.2", optional = true }sphinx-automodapi = { version = "0.16.0", optional = true }+rich = "^13.7.1"[tool.poetry.group.test.dependencies]