DSPy: The framework for programming—not prompting—language models
Python
remove experimental module (#8391)
TomuHirata3ded7a626651
diff --git a/dspy/experimental/__init__.py b/dspy/experimental/__init__.pydeleted file mode 100644index 25cdf7b7b..000000000--- a/dspy/experimental/__init__.py+++ /dev/null@@ -1,3 +0,0 @@-from dspy.experimental.module_graph import *-from dspy.experimental.synthesizer import *-from dspy.experimental.synthetic_data import *diff --git a/dspy/experimental/module_graph.py b/dspy/experimental/module_graph.pydeleted file mode 100644index 98e53f6ed..000000000--- a/dspy/experimental/module_graph.py+++ /dev/null@@ -1,151 +0,0 @@-import dspy--try:- import graphviz # type: ignore-- graphviz_available = True-except ImportError:- graphviz_available = False---class ModuleGraph:- def __init__(self, module_name, module):- if graphviz_available is False:- raise ImportError(- """Please install graphviz to use this feature.- Run 'pip install graphviz'"""- )-- self.graph = graphviz.Digraph(format="png")- self.nodes = set()- self.module_name = module_name- self.module = module- self.inspect_settings(dspy.settings)- self.add_module(self.module_name, self.module)-- def inspect_settings(self, settings):- """Check for the existence and configuration of LM and RM and add them to the graph."""- components = {"lm": settings.lm, "rm": settings.rm}- for component_name, component in components.items():- if component:- details = {- attr: getattr(component, attr)- for attr in dir(component)- if not attr.startswith("_") and not callable(getattr(component, attr))- }- component_details = f"{component_name.upper()} Details: " + ", ".join(- f"{k}: {v}" for k, v in details.items() if k != "history"- )- self.graph.node(component_name, label=component_details, shape="box")- self.nodes.add(component_name)-- def add_module(self, module_name, module):- """Add a module to the graph"""-- module_type = type(module)-- if "dspy.predict" in str(module_type):- module_name = self.generate_module_name(module_name, module_type)- self.process_submodule(module_name, module)- else:- self.process_submodules(module_name, module)-- def generate_module_name(self, base_name, module_type):- """Generate a module name based on the module type"""- type_map = {- "Predict": "__Predict",- "ReAct": "__ReAct",- "ChainOfThought": "__ChainOfThought",- "ProgramOfThought": "__ProgramOfThought",- "MultiChainComparison": "__MultiChainComparison",- "majority": "__majority",- }-- for key, suffix in type_map.items():- if key in str(module_type):- return base_name + suffix- return base_name-- def process_submodules(self, module_name, module):- """Process submodules of a module and add them to the graph"""-- for sub_module_name, sub_module in module.__dict__.items():- if isinstance(sub_module, dspy.Predict):- sub_module_name = self.generate_module_name(sub_module_name, type(sub_module))- self.process_submodule(f"{module_name}__{sub_module_name}", sub_module)- elif isinstance(sub_module, (dspy.Module, dspy.Retrieve)):- self.add_module(f"{module_name}__{sub_module_name}", sub_module)- if isinstance(sub_module, dspy.Retrieve):- self.graph.edge("rm", "lm", label="RM used in Module")-- def process_submodule(self, sub_module_name, sub_module):- """Process a submodule and add it to the graph"""-- for field_type, fields in [- ("input", sub_module.signature.input_fields),- ("output", sub_module.signature.output_fields),- ]:- for field_name, field in fields.items():- node_id = f"{sub_module_name}_{field_type}_{field_name}"- if node_id not in self.nodes:- label = f"{field_name}: ({field.json_schema_extra['desc']})"- self.graph.node(node_id, label=label, shape="ellipse")- self.nodes.add(node_id)- edge_direction = (node_id, sub_module_name) if field_type == "input" else (sub_module_name, node_id)- self.graph.edge(*edge_direction)-- # Add node for the submodule itself- self.graph.node(sub_module_name, label=sub_module_name, shape="box")- self.nodes.add(sub_module_name)-- # Connect submodule to LM if configured- if "lm" in self.nodes:- self.graph.edge("lm", sub_module_name, label="LM used in Module")-- def render_graph(self, filename=None):- """Render the graph to a file(png)"""- if filename is None:- filename = self.module_name- self.graph.render(filename)---# Example usage of the ModuleGraph class:-# import dspy-# import os-# from dotenv import load_dotenv-# from dspy.experimental import ModuleGraph--# load_dotenv()--# # Configuration of dspy models-# llm = dspy.OpenAI(-# model='gpt-3.5-turbo',-# api_key=os.environ['OPENAI_API_KEY'],-# max_tokens=100-# )--# colbertv2_wiki = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')--# dspy.settings.configure(lm=llm, rm=colbertv2_wiki)--# class GenerateAnswer(dspy.Signature):-# "Answer with long and detailed answers"-# context = dspy.InputField(desc="may content relevant facts")-# question = dspy.InputField()-# answer = dspy.OutputField(desc="often between 10 and 50 words")--# class RAG(dspy.Module):-# def __init__(self, num_passages=3):-# super().__init__()-# self.retrieve = dspy.Retrieve(k=num_passages)-# self.generate_answer = dspy.ChainOfThought(GenerateAnswer)--# def forward(self, question):-# context = self.retrieve(question).passages-# prediction = self.generate_answer(context=context, question=question)-# return dspy.Prediction(context=context, answer=prediction.answer)--# rag_system = RAG()-# graph = ModuleGraph("RAG", rag_system)--# graph.render_graph()diff --git a/dspy/experimental/synthesizer/__init__.py b/dspy/experimental/synthesizer/__init__.pydeleted file mode 100644index 861a14ce9..000000000--- a/dspy/experimental/synthesizer/__init__.py+++ /dev/null@@ -1,7 +0,0 @@-from dspy.experimental.synthesizer.config import SynthesizerArguments-from dspy.experimental.synthesizer.synthesizer import Synthesizer--__all__ = [- "Synthesizer",- "SynthesizerArguments",-]diff --git a/dspy/experimental/synthesizer/config.py b/dspy/experimental/synthesizer/config.pydeleted file mode 100644index ff2f04f14..000000000--- a/dspy/experimental/synthesizer/config.py+++ /dev/null@@ -1,24 +0,0 @@-from typing import Any, Optional--from pydantic import BaseModel, model_validator---class SynthesizerArguments(BaseModel):- feedback_mode: Optional[str] = None- num_example_for_feedback: Optional[int] = None-- input_lm_model: Optional[Any] = None- output_lm_model: Optional[Any] = None- output_teacher_module: Optional[Any] = None-- num_example_for_optim: Optional[int] = None-- @model_validator(mode="after")- def validate_feedback_mode(self):- if self.feedback_mode and self.feedback_mode not in ["human", "llm"]:- raise ValueError("Feedback mode should be either 'human' or 'llm'.")-- if self.feedback_mode and not self.num_example_for_feedback:- raise ValueError("Number of examples for feedback is required when feedback mode is provided.")-- return selfdiff --git a/dspy/experimental/synthesizer/instruction_suffixes.py b/dspy/experimental/synthesizer/instruction_suffixes.pydeleted file mode 100644index 753550bd4..000000000--- a/dspy/experimental/synthesizer/instruction_suffixes.py+++ /dev/null@@ -1,3 +0,0 @@-INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX = """\n\nI'll also be providing you some data I generated before hand, make sure the data you generate if consistent with task I provided but different from the data I provided in every way possible."""--INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX = "\n\nAdditionally, I'll be providing you with feedback on the data you generate, while generating the data make sure to take into account the feedback I provide and try to improve the data you generate based on the feedback I provide."diff --git a/dspy/experimental/synthesizer/signatures.py b/dspy/experimental/synthesizer/signatures.pydeleted file mode 100644index 022b1de95..000000000--- a/dspy/experimental/synthesizer/signatures.py+++ /dev/null@@ -1,102 +0,0 @@-import dspy--from .utils import format_examples---class UnderstandTask(dspy.Signature):- """I'll be providing you a task description. Your task is to prepare a concise, comprehensible summary that captures the broad essence and purpose of this task description. Your summary should illuminate the general objective and the type of problem being solved, offering a clear picture of what the task entails at a high level. Avoid getting into the nuances or specifics of individual datapoints, models, examples, algorithms, or any intricate technicalities. Your explanation should serve to clarify the task's overall goal and its basic premise without touching on methodologies or solutions."""-- task_description = dspy.InputField(- prefix="Task Description:",- desc="Description of the task.",- )- explanation = dspy.OutputField(- prefix="Task Description:",- desc="Explanation of the task.",- )---class ExplainTask(dspy.Signature):- """Analyze the provided set of datapoints carefully, and prepare a concise, comprehensible summary that captures the broad essence and purpose of the task these datapoints aim to address. Your summary should illuminate the general objective and the type of problem being solved, offering a clear picture of what the task entails at a high level. Avoid getting into the nuances of individual datapoints, specifics about models, examples, algorithms, or any intricate technicalities. Your explanation should serve to clarify the task's overall goal and its basic premise, without touching on methodologies or solutions."""-- examples = dspy.InputField(- prefix="Examples Datapoints:",- desc="List of datapoints to analyze and explain the task.",- format=format_examples,- )- explanation = dspy.OutputField(- prefix="Task Description:",- desc="Explanation of the task.",- )---class UpdateTaskDescriptionBasedOnFeedback(dspy.Signature):- """Update the task description based on the feedback provided. Ensure that the revised task description incorporates the feedback to improve its overall clarity and effectiveness. Focus on enhancing the task's goal and basic premise, without delving into specific data points, models, examples, algorithms, or technical intricacies. Your explanation should aim to clarify the task's fundamental objective and purpose."""-- task_description = dspy.InputField(- prefix="Task Description:",- desc="Description of the task.",- )- feedback = dspy.InputField(- prefix="Feedback:",- desc="Feedback on the task description.",- )- updated_task_description = dspy.OutputField(- prefix="Task Description:",- desc="Updated description of the task.",- )---class GetFeedbackOnGeneration(dspy.Signature):- """Provide constructive feedback on the synthetic data generated, focusing on its quality, relevance, and diversity. Highlight any areas that require improvement and offer suggestions for enhancement. The feedback should center on the overall effectiveness of the synthetic data in aligning with the task description and knowledge seed. Avoid delving into specific data points, models, examples, algorithms, or technical intricacies. Your feedback should be critical but constructive, aiming to improve the synthetic data and the task description."""-- synthetic_data = dspy.InputField(- prefix="Synthetic Data:",- desc="Synthetic data generated.",- format=format_examples,- )- task_description = dspy.InputField(- prefix="Task Description:",- desc="Description of the task the synthetic data is aligned with.",- )- feedback = dspy.OutputField(- prefix="Feedback:",- desc="Feedback on the synthetic data.",- )---class GenerateFieldDescription(dspy.Signature):- """Generate a concise and informative description for a given field based on the provided name and task description. This description should be no longer than 10 words and should be in simple english."""-- task_description = dspy.InputField(- prefix="Task Description:",- desc="Description of the task the field is an input to.",- )- field_name = dspy.InputField(- prefix="Field Name:",- desc="Name of the field to generate synthetic data for.",- )- field_description = dspy.OutputField(- prefix="Field Description:",- desc="Description of the field.",- )---class GenerateInputFieldsData(dspy.Signature):- """Create synthetic data using the task description and the provided knowledge seed. Your task is to generate diverse and imaginative data that aligns with the given task description and knowledge seed. You are encouraged to be creative and not limit yourself, allowing for a wide range of synthetic data that reflects the characteristics and details provided in the task description. The data should be unique and varied, showcasing originality and creativity while maintaining relevance to the task and knowledge seed.-- A knowledge seed is the index of the knowledge base you have, each index represents a different knowledge base."""-- knowledge_seed = dspy.InputField(- prefix="Knowledge Seed:",- desc="Seed for the knowledge base search to base the inputs around.",- format=lambda x: str(x),- )- task_description = dspy.InputField(- prefix="Task Description:",- desc="Description of the task the field is an input to.",- )---class GenerateOutputFieldsData(dspy.Signature):- passdiff --git a/dspy/experimental/synthesizer/synthesizer.py b/dspy/experimental/synthesizer/synthesizer.pydeleted file mode 100644index 666074ba8..000000000--- a/dspy/experimental/synthesizer/synthesizer.py+++ /dev/null@@ -1,254 +0,0 @@-import random-from collections.abc import Mapping-from typing import List, Optional, Union--from rich import print as rprint-from tqdm import tqdm, trange--import dspy-from dspy.experimental.synthesizer.config import SynthesizerArguments-from dspy.experimental.synthesizer.instruction_suffixes import (- INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX,- INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX,-)-from dspy.experimental.synthesizer.signatures import (- ExplainTask,- GenerateFieldDescription,- GenerateInputFieldsData,- GenerateOutputFieldsData,- GetFeedbackOnGeneration,- UnderstandTask,- UpdateTaskDescriptionBasedOnFeedback,-)-from dspy.experimental.synthesizer.utils import format_examples---class Synthesizer:- def __init__(self, config: SynthesizerArguments):- self.config = config- self.input_lm = config.input_lm_model or dspy.settings.lm- self.output_lm = config.output_lm_model or dspy.settings.lm-- self.explain_task = dspy.Predict(ExplainTask)- self.understand_task = dspy.Predict(UnderstandTask)- self.get_feedback_on_generation = dspy.Predict(GetFeedbackOnGeneration)- self.generate_field_description = dspy.Predict(GenerateFieldDescription)- self.update_task_description = dspy.Predict(UpdateTaskDescriptionBasedOnFeedback)-- self.generate_input_data = GenerateInputFieldsData- self.generate_output_data = GenerateOutputFieldsData-- def _gather_feedback(self, examples: dspy.Example) -> str:- if self.config.feedback_mode == "human":- input_keys = examples.inputs().keys()-- print("-" * 75)- print_text = "[bold blue]Generated Data:[bold blue]\n[bold red]Inputs:[bold red]\n"-- for key in input_keys:- print_text += f"\t[bold yellow]{key}[bold yellow]: [green]{examples[key]}[green]\n"-- rprint(print_text)- feedback = input("Provide feedback on the generated data: ")- print("-" * 75)-- return feedback-- elif self.config.feedback_mode == "llm":- feedback = self.get_feedback_on_generation(- synthetic_data=[examples],- task_description=self.generate_output_data.__doc__,- )-- return feedback.feedback-- else:- raise ValueError("Feedback mode should be either 'human' or 'llm'.")-- def _get_field_data(self, key: str, keys_dict: Mapping[str, str]):- if key.startswith("$"):- field_details = self.generate_field_description(- task_description=keys_dict["task_description"],- field_name=key,- )-- field_name = key- field_description = field_details.field_description-- return field_name, field_description-- else:- field_name = key- field_description = keys_dict[key]-- return field_name, field_description-- def _prepare_synthetic_data_predictors(- self,- input_keys: Mapping[str, str],- output_keys: Mapping[str, str],- ground_source: Optional[Union[List[dspy.Example], dspy.Signature]] = None,- ):- for key in tqdm(input_keys, desc="Preparing Input Fields"):- field_name, field_description = self._get_field_data(key, input_keys)-- output_field = dspy.OutputField(- prefix=f"{field_name}:",- desc=field_description,- )- self.generate_input_data = self.generate_input_data.insert(- -1,- field_name,- output_field,- )-- if ground_source:- self.generate_input_data = self.generate_input_data.insert(- -1,- "ground_source",- dspy.InputField(- prefix="Pre-Generated Examples:",- desc="Pre-Generated Examples to differ the inputs around.",- format=format_examples,- ),- )-- input_field = dspy.InputField(- prefix=f"{field_name}:",- desc=field_description,- )- self.generate_output_data = self.generate_output_data.insert(- -1,- field_name,- input_field,- )-- for key in tqdm(output_keys, desc="Preparing Output Fields"):- field_name, field_description = self._get_field_data(key, output_keys)-- output_field = dspy.OutputField(- prefix=f"{field_name}:",- desc=field_description,- )- self.generate_output_data = self.generate_output_data.insert(- -1,- field_name,- output_field,- )-- return dspy.ChainOfThought(self.generate_input_data), dspy.Predict(self.generate_output_data)-- def _get_dataset_metadata(self, ground_source: Union[List[dspy.Example], dspy.Signature]):- if isinstance(ground_source, dspy.SignatureMeta):- task_description = ground_source.__doc__- if task_description.startswith("Given the fields"):- task_description = self.understand_task(examples=ground_source.__doc__).explanation-- input_keys = {k: v.json_schema_extra["desc"] for k, v in ground_source.input_fields.items()}- output_keys = {k: v.json_schema_extra["desc"] for k, v in ground_source.output_fields.items()}-- return task_description, input_keys, output_keys-- elif isinstance(ground_source, list) and isinstance(ground_source[0], dspy.Example):- task_description = self.explain_task(examples=ground_source).explanation- input_keys = {key: f"${{{key}}}" for key in ground_source[0].inputs()}- output_keys = {key: f"${{{key}}}" for key in ground_source[0].labels()}-- return task_description, input_keys, output_keys-- else:- raise ValueError("Ground source must be either a list of examples or a signature.")-- def generate(- self,- ground_source: Union[List[dspy.Example], dspy.Signature],- num_data: int,- batch_size: int = 1,- ):- batch_size = batch_size or 1- task_description, input_keys, output_keys = self._get_dataset_metadata(ground_source)-- if self.config.num_example_for_optim:- self.generate_input_data.__doc__ += INPUT_GENERATION_TASK_WITH_EXAMPLES_SUFFIX-- if self.config.feedback_mode:- self.generate_input_data.__doc__ += INPUT_GENERATION_TASK_WITH_FEEDBACK_SUFFIX-- self.generate_output_data.__doc__ = task_description-- self.input_predictor, self.output_predictor = self._prepare_synthetic_data_predictors(- input_keys=input_keys,- output_keys=output_keys,- ground_source=ground_source if self.config.num_example_for_optim else None,- )-- data = []- feedback = ""-- for idx in trange(0, num_data, batch_size, desc="Generating Synthetic Data"):- iter_temperature = 0.7 + 0.01 * idx- iter_seed = random.randint(0, 1000000)-- kwargs = {- "task_description": task_description,- "knowledge_seed": iter_seed,- "config": {"temperature": iter_temperature, "n": batch_size},- }-- if self.config.num_example_for_optim:- if not isinstance(ground_source, list):- raise ValueError(- "Ground source must be a list of examples when `num_example_for_optim` is provided."- )- kwargs["ground_source"] = random.sample(ground_source, k=self.config.num_example_for_optim)-- with dspy.context(lm=self.input_lm):- inputs = self.input_predictor(**kwargs)-- input_kwargs = [- {key: getattr(completions, key) for key in input_keys} for completions in inputs.completions- ]-- for kwargs in input_kwargs:- outputs = None-- with dspy.context(lm=self.output_lm, temperature=iter_temperature):- if self.config.output_teacher_module:- outputs = self.config.output_teacher_module(**kwargs)-- else:- outputs = self.output_predictor(**kwargs, config={"temperature": iter_temperature})-- output_kwargs = {key: getattr(outputs, key) for key in output_keys}-- data.append(dspy.Example(**kwargs, **output_kwargs).with_inputs(*input_keys))-- if self.config.feedback_mode and idx < self.config.num_example_for_feedback:- feedback = self._gather_feedback(data[-1])-- task_description = self.update_task_description(- task_description=task_description,- feedback=feedback,- ).updated_task_description-- self.output_predictor.signature.__doc__ = task_description-- return data-- def export(self, data: List[dspy.Example], path: str, mode: Optional[str] = None, **kwargs):- extension = mode or path.split(".")[-1]-- from datasets import Dataset-- dataset = Dataset.from_list(- [example.toDict() for example in data],- )-- if extension == "csv":- dataset.to_csv(path_or_buf=path, **kwargs)-- elif extension == "json":- dataset.to_json(path_or_buf=path, **kwargs)-- elif extension == "arrow" or extension == "hf":- dataset.save_to_disk(path)diff --git a/dspy/experimental/synthesizer/utils.py b/dspy/experimental/synthesizer/utils.pydeleted file mode 100644index 8e92fc097..000000000--- a/dspy/experimental/synthesizer/utils.py+++ /dev/null@@ -1,24 +0,0 @@-from typing import List--import dspy---def format_examples(examples: List[dspy.Example]) -> str:- if isinstance(examples, str):- return examples-- formatted_example = ""-- for example in examples:- input_keys = example.inputs().keys()- label_keys = example.labels().keys()-- formatted_example += "Inputs:\n"- for key in input_keys:- formatted_example += f"{key}: {example[key]}\n"-- formatted_example += "Outputs:\n"- for key in label_keys:- formatted_example += f"{key}: {example[key]}\n"-- return formatted_examplediff --git a/dspy/experimental/synthetic_data.py b/dspy/experimental/synthetic_data.pydeleted file mode 100644index 197dcc4e4..000000000--- a/dspy/experimental/synthetic_data.py+++ /dev/null@@ -1,106 +0,0 @@-import logging-import random-from typing import List, Optional--from pydantic import BaseModel--import dspy---class DescriptionSignature(dspy.Signature):- field_name = dspy.InputField(desc="name of a field")- example = dspy.InputField(desc="an example value for the field")- description = dspy.OutputField(desc="a short text only description of what the field contains")---class SyntheticDataGenerator:- def __init__(self, schema_class: Optional[BaseModel] = None, examples: Optional[List[dspy.Example]] = None):- self.schema_class = schema_class- self.examples = examples-- def generate(self, sample_size: int) -> List[dspy.Example]:- """Generate synthetic examples.-- Args:- sample_size (int): number of examples to generate- Raises:- ValueError: either a schema_class or examples should be provided- Returns:- List[dspy.Example]: list of synthetic examples generated- """- if not self.schema_class and not self.examples:- raise ValueError("Either a schema_class or examples must be provided.")- if self.examples and len(self.examples) >= sample_size:- logging.info("No additional data generation needed.")- return self.examples[:sample_size]-- additional_samples_needed = sample_size - (len(self.examples) if self.examples else 0)- generated_examples = self._generate_additional_examples(additional_samples_needed)-- return self.examples + generated_examples if self.examples else generated_examples-- def _define_or_infer_fields(self):- """Define fields to generate if a schema class is provided.- Infer fields to generate if an initial sample of examples is provided.-- Returns:- dict: dictionary of fields to generate- """- if self.schema_class:- data_schema = self.schema_class.model_json_schema()- properties = data_schema["properties"]- elif self.examples:- inferred_schema = self.examples[0].__dict__["_store"]- descriptor = dspy.Predict(DescriptionSignature)- properties = {- field: {- "description": str((descriptor(field_name=field, example=str(inferred_schema[field]))).description)- }- for field in inferred_schema.keys()- }- else:- properties = {}- return properties-- def _generate_additional_examples(self, additional_samples_needed: int) -> List[dspy.Example]:- """Generate additional examples if needed.-- Args:- additional_samples_needed (int): the difference between the desired- number of examples and the current number of examples- Returns:- List[dspy.Example]: list of synthetic examples- """- properties = self._define_or_infer_fields()- class_name = f"{self.schema_class.__name__ if self.schema_class else 'Inferred'}Signature"- fields = self._prepare_fields(properties)-- signature_class = type(class_name, (dspy.Signature,), fields)- generator = dspy.Predict(signature_class, n=additional_samples_needed)- response = generator(sindex=str(random.randint(1, additional_samples_needed)))-- return [- dspy.Example({field_name: getattr(completion, field_name) for field_name in properties.keys()})- for completion in response.completions- ]-- def _prepare_fields(self, properties) -> dict:- """Prepare fields to generate in an appropriate format."""- return {- "__doc__": f"Generates the following outputs: {{{', '.join(properties.keys())}}}.",- "sindex": dspy.InputField(desc="a random string"),- **{- field_name: dspy.OutputField(desc=properties[field_name].get("description", "No description"))- for field_name in properties.keys()- },- }---# # Usage example-# # Generating synthetic data via a pydantic model-# generator = SyntheticDataGenerator(schema_class=SyntheticFacts)-# examples = generator.generate(sample_size=6)--# # Generating synthetic data via existing examples-# generator = SyntheticDataGenerator(examples=existing_examples)-# examples = generator.generate(sample_size=5)