DSPy: The framework for programming—not prompting—language models
Python
Add custom instruction_proposer support to GEPA with multimodal (dspy.Image) handling (#8737)
Andres Serrano91e7e23bf977
diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.pyindex a4ea0878c..14aeac89e 100644--- a/dspy/teleprompt/gepa/gepa.py+++ b/dspy/teleprompt/gepa/gepa.py@@ -2,17 +2,16 @@ import inspectimport loggingimport randomfrom dataclasses import dataclass-from typing import TYPE_CHECKING, Any, Literal, Optional, Protocol, Union+from typing import Any, Literal, Optional, Protocol, Union++from gepa import GEPAResult+from gepa.core.adapter import ProposalFnfrom dspy.clients.lm import LMfrom dspy.primitives import Example, Module, Prediction+from dspy.teleprompt.gepa.gepa_utils import DspyAdapter, DSPyTrace, PredictorFeedbackFn, ScoreWithFeedbackfrom dspy.teleprompt.teleprompt import Teleprompter-if TYPE_CHECKING:- from gepa import GEPAResult-- from dspy.teleprompt.gepa.gepa_utils import DspyAdapter, DSPyTrace, PredictorFeedbackFn, ScoreWithFeedback-logger = logging.getLogger(__name__)AUTO_RUN_SETTINGS = {@@ -210,6 +209,18 @@ class GEPA(Teleprompter):a strong reflection model. Consider using `dspy.LM(model='gpt-5', temperature=1.0, max_tokens=32000)`for optimal performance.skip_perfect_score: Whether to skip examples with perfect scores during reflection. Default is True.+ instruction_proposer: Optional custom instruction proposer implementing GEPA's ProposalFn protocol.+ If provided, GEPA will use this custom proposer instead of its default instruction proposal+ mechanism to generate improved instructions based on feedback from failed examples. This is+ particularly useful when you need specialized instruction generation for multimodal inputs+ (like dspy.Image) or custom types. Use `MultiModalInstructionProposer()` from+ `dspy.teleprompt.gepa.instruction_proposal` for handling visual content. If None (default),+ GEPA uses its built-in text-optimized proposer (see `gepa.strategies.instruction_proposal.InstructionProposalSignature`+ for reference implementation).++ Note: When both instruction_proposer and reflection_lm are set, the instruction_proposer is called+ in the reflection_lm context. However, reflection_lm is optional when using a custom instruction_proposer.+ Custom instruction proposers can invoke their own LLMs if needed.add_format_failure_as_feedback: Whether to add format failures as feedback. Default is False.use_merge: Whether to use merge-based optimization. Default is True.max_merge_invocations: The maximum number of merge invocations to perform. Default is 5.@@ -260,12 +271,13 @@ class GEPA(Teleprompter):auto: Literal["light", "medium", "heavy"] | None = None,max_full_evals: int | None = None,max_metric_calls: int | None = None,- # Reflection based configuration+ # Reflection configurationreflection_minibatch_size: int = 3,candidate_selection_strategy: Literal["pareto", "current_best"] = "pareto",reflection_lm: LM | None = None,skip_perfect_score: bool = True,add_format_failure_as_feedback: bool = False,+ instruction_proposer: "ProposalFn | None" = None,# Merge-based configurationuse_merge: bool = True,max_merge_invocations: int | None = 5,@@ -309,12 +321,17 @@ class GEPA(Teleprompter):self.max_full_evals = max_full_evalsself.max_metric_calls = max_metric_calls- # Reflection based configuration+ # Reflection configurationself.reflection_minibatch_size = reflection_minibatch_sizeself.candidate_selection_strategy = candidate_selection_strategy- # self.reflection_lm = reflection_lm- assert reflection_lm is not None, "GEPA requires a reflection language model to be provided. Typically, you can use `dspy.LM(model='gpt-5', temperature=1.0, max_tokens=32000)` to get a good reflection model. Reflection LM is used by GEPA to reflect on the behavior of the program and propose new instructions, and will benefit from a strong model."- self.reflection_lm = lambda x: reflection_lm(x)[0]++ assert reflection_lm is not None or instruction_proposer is not None, (+ "GEPA requires a reflection language model, or custom instruction proposer to be provided. "+ "Typically, you can use `dspy.LM(model='gpt-5', temperature=1.0, max_tokens=32000)` to get a good reflection model. "+ "Reflection LM is used by GEPA to reflect on the behavior of the program and propose new instructions, and will benefit from a strong model. "+ )++ self.reflection_lm = reflection_lmself.skip_perfect_score = skip_perfect_scoreself.add_format_failure_as_feedback = add_format_failure_as_feedback@@ -341,6 +358,8 @@ class GEPA(Teleprompter):# Reproducibilityself.seed = seed+ self.custom_instruction_proposer = instruction_proposer+def auto_budget(self, num_preds, num_candidates, valset_size: int, minibatch_size: int = 35, full_eval_steps: int = 5) -> int:import numpy as npnum_trials = int(max(2 * (num_preds * 2) * np.log2(num_candidates), 1.5 * num_candidates))@@ -452,10 +471,10 @@ class GEPA(Teleprompter):num_threads=self.num_threads,add_format_failure_as_feedback=self.add_format_failure_as_feedback,rng=rng,+ reflection_lm=self.reflection_lm,+ custom_instruction_proposer=self.custom_instruction_proposer,)- reflection_lm = self.reflection_lm-# Instantiate GEPA with the simpler adapter-based APIbase_program = {name: pred.signature.instructions for name, pred in student.named_predictors()}gepa_result: GEPAResult = optimize(@@ -465,7 +484,7 @@ class GEPA(Teleprompter):adapter=adapter,# Reflection-based configuration- reflection_lm=reflection_lm,+ reflection_lm=(lambda x: self.reflection_lm(x)[0]) if self.reflection_lm is not None else None,candidate_selection_strategy=self.candidate_selection_strategy,skip_perfect_score=self.skip_perfect_score,reflection_minibatch_size=self.reflection_minibatch_size,diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.pyindex 95d1fc17f..66b3b4636 100644--- a/dspy/teleprompt/gepa/gepa_utils.py+++ b/dspy/teleprompt/gepa/gepa_utils.py@@ -3,9 +3,12 @@ import randomfrom typing import Any, Callable, Protocolfrom gepa import EvaluationBatch, GEPAAdapter+from gepa.core.adapter import ProposalFn+import dspyfrom dspy.adapters.chat_adapter import ChatAdapterfrom dspy.adapters.types import History+from dspy.adapters.types.base_type import Typefrom dspy.evaluate import Evaluatefrom dspy.primitives import Example, Predictionfrom dspy.teleprompt.bootstrap_trace import TraceData@@ -58,6 +61,8 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):num_threads: int | None = None,add_format_failure_as_feedback: bool = False,rng: random.Random | None = None,+ reflection_lm=None,+ custom_instruction_proposer: "ProposalFn | None" = None,):self.student = student_moduleself.metric_fn = metric_fn@@ -66,6 +71,34 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):self.num_threads = num_threadsself.add_format_failure_as_feedback = add_format_failure_as_feedbackself.rng = rng or random.Random(0)+ self.reflection_lm = reflection_lm+ self.custom_instruction_proposer = custom_instruction_proposer++ if self.custom_instruction_proposer is not None:+ # We are only overriding the propose_new_texts method when a custom+ # instruction proposer is provided. Otherwise, we use the GEPA+ # default propose_new_texts.++ def custom_propose_new_texts(+ candidate: dict[str, str],+ reflective_dataset: dict[str, list[dict[str, Any]]],+ components_to_update: list[str]+ ) -> dict[str, str]:+ if self.reflection_lm is not None:+ with dspy.context(lm=self.reflection_lm):+ return self.custom_instruction_proposer(+ candidate=candidate,+ reflective_dataset=reflective_dataset,+ components_to_update=components_to_update+ )+ else:+ return self.custom_instruction_proposer(+ candidate=candidate,+ reflective_dataset=reflective_dataset,+ components_to_update=components_to_update+ )++ self.propose_new_texts = custom_propose_new_texts# Cache predictor names/signaturesself.named_predictors = list(self.student.named_predictors())@@ -184,7 +217,12 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):for input_key, input_val in inputs.items():if contains_history and input_key == history_key_name:continue- new_inputs[input_key] = str(input_val)++ if isinstance(input_val, Type) and self.custom_instruction_proposer is not None:+ # Keep original object - will be properly formatted when sent to reflection LM+ new_inputs[input_key] = input_val+ else:+ new_inputs[input_key] = str(input_val)if isinstance(outputs, FailedPrediction):s = "Couldn't parse the output as per the expected output format. The model's raw response was:\n"diff --git a/dspy/teleprompt/gepa/instruction_proposal.py b/dspy/teleprompt/gepa/instruction_proposal.pynew file mode 100644index 000000000..23b7ab180--- /dev/null+++ b/dspy/teleprompt/gepa/instruction_proposal.py@@ -0,0 +1,311 @@+from typing import Any++from gepa.core.adapter import ProposalFn++import dspy+from dspy.adapters.types.base_type import Type+++class GenerateEnhancedMultimodalInstructionFromFeedback(dspy.Signature):+ """I provided an assistant with instructions to perform a task involving visual content, but the assistant's performance needs improvement based on the examples and feedback below.++ Your task is to write a better instruction for the assistant that addresses the specific issues identified in the feedback, with particular attention to how visual and textual information should be analyzed and integrated.++ ## Analysis Steps:+ 1. **Read the inputs carefully** and identify both the visual and textual input formats, understanding how they work together+ 2. **Read all the assistant responses and corresponding feedback** to understand what went wrong with visual analysis, text processing, or their integration+ 3. **Identify visual analysis patterns** - what visual features, relationships, or details are important for this task+ 4. **Identify domain-specific knowledge** about both visual and textual aspects, as this information may not be available to the assistant in the future+ 5. **Look for successful visual-textual integration strategies** and include these patterns in the instruction+ 6. **Address specific visual analysis issues** mentioned in the feedback++ ## Instruction Requirements:+ - **Clear task definition** explaining how to process both visual and textual inputs+ - **Visual analysis guidance** specific to this task (what to look for, how to describe, what features matter)+ - **Integration strategies** for combining visual observations with textual information+ - **Domain-specific knowledge** about visual concepts, terminology, or relationships+ - **Error prevention guidance** for common visual analysis mistakes shown in the feedback+ - **Precise, actionable language** for both visual and textual processing++ Focus on creating an instruction that helps the assistant properly analyze visual content, integrate it with textual information, and avoid the specific visual analysis mistakes shown in the examples."""++ current_instruction = dspy.InputField(+ desc="The current instruction that was provided to the assistant to perform the multimodal task"+ )+ examples_with_feedback = dspy.InputField(+ desc="Task examples with visual content showing inputs, assistant outputs, and feedback. "+ "Pay special attention to feedback about visual analysis accuracy, visual-textual integration, "+ "and any domain-specific visual knowledge that the assistant missed."+ )++ improved_instruction = dspy.OutputField(+ desc="A better instruction for the assistant that addresses visual analysis issues, provides "+ "clear guidance on how to process and integrate visual and textual information, includes "+ "necessary visual domain knowledge, and prevents the visual analysis mistakes shown in the examples."+ )+++class SingleComponentMultiModalProposer(dspy.Module):+ """+ dspy.Module for proposing improved instructions based on feedback.+ """++ def __init__(self):+ super().__init__()+ self.propose_instruction = dspy.Predict(GenerateEnhancedMultimodalInstructionFromFeedback)++ def forward(self, current_instruction: str, reflective_dataset: list[dict[str, Any]]) -> str:+ """+ Generate an improved instruction based on current instruction and feedback examples.++ Args:+ current_instruction: The current instruction that needs improvement+ reflective_dataset: List of examples with inputs, outputs, and feedback+ May contain dspy.Image objects in inputs++ Returns:+ str: Improved instruction text+ """+ # Format examples with enhanced pattern recognition+ formatted_examples, image_map = self._format_examples_with_pattern_analysis(reflective_dataset)++ # Build kwargs for the prediction call+ predict_kwargs = {+ "current_instruction": current_instruction,+ "examples_with_feedback": formatted_examples,+ }++ # Create a rich multimodal examples_with_feedback that includes both text and images+ predict_kwargs["examples_with_feedback"] = self._create_multimodal_examples(formatted_examples, image_map)++ # Use current dspy LM settings (GEPA will pass reflection_lm via context)+ result = self.propose_instruction(**predict_kwargs)++ return result.improved_instruction++ def _format_examples_with_pattern_analysis(+ self, reflective_dataset: list[dict[str, Any]]+ ) -> tuple[str, dict[int, list[Type]]]:+ """+ Format examples with pattern analysis and feedback categorization.++ Returns:+ tuple: (formatted_text_with_patterns, image_map)+ """+ # First, use the existing proven formatting approach+ formatted_examples, image_map = self._format_examples_for_instruction_generation(reflective_dataset)++ # Enhanced analysis: categorize feedback patterns+ feedback_analysis = self._analyze_feedback_patterns(reflective_dataset)++ # Add pattern analysis to the formatted examples+ if feedback_analysis["summary"]:+ pattern_summary = self._create_pattern_summary(feedback_analysis)+ enhanced_examples = f"{pattern_summary}\n\n{formatted_examples}"+ return enhanced_examples, image_map++ return formatted_examples, image_map++ def _analyze_feedback_patterns(self, reflective_dataset: list[dict[str, Any]]) -> dict[str, Any]:+ """+ Analyze feedback patterns to provide better context for instruction generation.++ Categorizes feedback into:+ - Error patterns: Common mistakes and their types+ - Success patterns: What worked well and should be preserved/emphasized+ - Domain knowledge gaps: Missing information that should be included+ - Task-specific guidance: Specific requirements or edge cases+ """+ analysis = {+ "error_patterns": [],+ "success_patterns": [],+ "domain_knowledge_gaps": [],+ "task_specific_guidance": [],+ "summary": "",+ }++ # Simple pattern recognition - could be enhanced further+ for example in reflective_dataset:+ feedback = example.get("Feedback", "").lower()++ # Identify error patterns+ if any(error_word in feedback for error_word in ["incorrect", "wrong", "error", "failed", "missing"]):+ analysis["error_patterns"].append(feedback)++ # Identify success patterns+ if any(+ success_word in feedback for success_word in ["correct", "good", "accurate", "well", "successfully"]+ ):+ analysis["success_patterns"].append(feedback)++ # Identify domain knowledge needs+ if any(+ knowledge_word in feedback+ for knowledge_word in ["should know", "domain", "specific", "context", "background"]+ ):+ analysis["domain_knowledge_gaps"].append(feedback)++ # Create summary if patterns were found+ if any(analysis[key] for key in ["error_patterns", "success_patterns", "domain_knowledge_gaps"]):+ analysis["summary"] = (+ f"Patterns identified: {len(analysis['error_patterns'])} error(s), {len(analysis['success_patterns'])} success(es), {len(analysis['domain_knowledge_gaps'])} knowledge gap(s)"+ )++ return analysis++ def _create_pattern_summary(self, feedback_analysis: dict[str, Any]) -> str:+ """Create a summary of feedback patterns to help guide instruction generation."""++ summary_parts = ["## Feedback Pattern Analysis\n"]++ if feedback_analysis["error_patterns"]:+ summary_parts.append(f"**Common Issues Found ({len(feedback_analysis['error_patterns'])} examples):**")+ summary_parts.append("Focus on preventing these types of mistakes in the new instruction.\n")++ if feedback_analysis["success_patterns"]:+ summary_parts.append(+ f"**Successful Approaches Found ({len(feedback_analysis['success_patterns'])} examples):**"+ )+ summary_parts.append("Build on these successful strategies in the new instruction.\n")++ if feedback_analysis["domain_knowledge_gaps"]:+ summary_parts.append(+ f"**Domain Knowledge Needs Identified ({len(feedback_analysis['domain_knowledge_gaps'])} examples):**"+ )+ summary_parts.append("Include this specialized knowledge in the new instruction.\n")++ return "\n".join(summary_parts)++ def _format_examples_for_instruction_generation(+ self, reflective_dataset: list[dict[str, Any]]+ ) -> tuple[str, dict[int, list[Type]]]:+ """+ Format examples using GEPA's markdown structure while preserving image objects.++ Returns:+ tuple: (formatted_text, image_map) where image_map maps example_index -> list[images]+ """++ def render_value_with_images(value, level=3, example_images=None):+ if example_images is None:+ example_images = []++ if isinstance(value, Type):+ image_idx = len(example_images) + 1+ example_images.append(value)+ return f"[IMAGE-{image_idx} - see visual content]\n\n"+ elif isinstance(value, dict):+ s = ""+ for k, v in value.items():+ s += f"{'#' * level} {k}\n"+ s += render_value_with_images(v, min(level + 1, 6), example_images)+ if not value:+ s += "\n"+ return s+ elif isinstance(value, (list, tuple)):+ s = ""+ for i, item in enumerate(value):+ s += f"{'#' * level} Item {i + 1}\n"+ s += render_value_with_images(item, min(level + 1, 6), example_images)+ if not value:+ s += "\n"+ return s+ else:+ return f"{str(value).strip()}\n\n"++ def convert_sample_to_markdown_with_images(sample, example_num):+ example_images = []+ s = f"# Example {example_num}\n"++ for key, val in sample.items():+ s += f"## {key}\n"+ s += render_value_with_images(val, level=3, example_images=example_images)++ return s, example_images++ formatted_parts = []+ image_map = {}++ for i, example_data in enumerate(reflective_dataset):+ formatted_example, example_images = convert_sample_to_markdown_with_images(example_data, i + 1)+ formatted_parts.append(formatted_example)++ if example_images:+ image_map[i] = example_images++ formatted_text = "\n\n".join(formatted_parts)++ if image_map:+ total_images = sum(len(imgs) for imgs in image_map.values())+ formatted_text = (+ f"The examples below include visual content ({total_images} images total). "+ "Please analyze both the text and visual elements when suggesting improvements.\n\n" + formatted_text+ )++ return formatted_text, image_map++ def _create_multimodal_examples(self, formatted_text: str, image_map: dict[int, list[Type]]) -> Any:+ """+ Create a multimodal input that contains both text and images for the reflection LM.++ Args:+ formatted_text: The formatted text with image placeholders+ image_map: Dictionary mapping example_index -> list[images] for structured access+ """+ if not image_map:+ return formatted_text++ # Collect all images from all examples+ all_images = []+ for example_images in image_map.values():+ all_images.extend(example_images)++ multimodal_content = [formatted_text]+ multimodal_content.extend(all_images)+ return multimodal_content+++class MultiModalInstructionProposer(ProposalFn):+ """GEPA-compatible multimodal instruction proposer.++ This class handles multimodal inputs (like dspy.Image) during GEPA optimization by using+ a single-component proposer for each component that needs to be updated.+ """++ def __init__(self):+ self.single_proposer = SingleComponentMultiModalProposer()++ def __call__(+ self,+ candidate: dict[str, str],+ reflective_dataset: dict[str, list[dict[str, Any]]],+ components_to_update: list[str],+ ) -> dict[str, str]:+ """GEPA-compatible proposal function.++ Args:+ candidate: Current component name -> instruction mapping+ reflective_dataset: Component name -> list of reflective examples+ components_to_update: List of component names to update++ Returns:+ dict: Component name -> new instruction mapping+ """+ updated_components = {}++ for component_name in components_to_update:+ if component_name in candidate and component_name in reflective_dataset:+ current_instruction = candidate[component_name]+ component_reflective_data = reflective_dataset[component_name]++ # Call the single-instruction proposer.+ #+ # In the future, proposals could consider multiple components instructions,+ # instead of just the current instruction, for more holistic instruction proposals.+ new_instruction = self.single_proposer(+ current_instruction=current_instruction, reflective_dataset=component_reflective_data+ )++ updated_components[component_name] = new_instruction++ return updated_componentsdiff --git a/pyproject.toml b/pyproject.tomlindex 0babb99b5..7dabe4d48 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -42,7 +42,7 @@ dependencies = ["rich>=13.7.1","numpy>=1.26.0","xxhash>=3.5.0",- "gepa[dspy]==0.0.7"+ "gepa[dspy]==0.0.8",][project.optional-dependencies]diff --git a/tests/teleprompt/test_gepa_instruction_proposer.py b/tests/teleprompt/test_gepa_instruction_proposer.pynew file mode 100644index 000000000..358e6d90f--- /dev/null+++ b/tests/teleprompt/test_gepa_instruction_proposer.py@@ -0,0 +1,353 @@+from dataclasses import dataclass+from typing import Any++import dspy+from dspy.teleprompt.gepa import instruction_proposal+from dspy.utils.dummies import DummyLM+++def count_messages_with_image_url_pattern(messages):+ """Helper to count image URLs in messages - borrowed from image adapter tests"""+ pattern = {"type": "image_url", "image_url": {"url": lambda x: isinstance(x, str)}}++ try:++ def check_pattern(obj, pattern):+ if isinstance(pattern, dict):+ if not isinstance(obj, dict):+ return False+ return all(k in obj and check_pattern(obj[k], v) for k, v in pattern.items())+ if callable(pattern):+ return pattern(obj)+ return obj == pattern++ def count_patterns(obj, pattern):+ count = 0+ if check_pattern(obj, pattern):+ count += 1+ if isinstance(obj, dict):+ count += sum(count_patterns(v, pattern) for v in obj.values())+ if isinstance(obj, (list, tuple)):+ count += sum(count_patterns(v, pattern) for v in obj)+ return count++ return count_patterns(messages, pattern)+ except Exception:+ return 0+++@dataclass+class ImagesInHistory:+ has_structured_images: bool+ has_text_serialized_images: bool+++def check_images_in_history(history: list[Any]) -> ImagesInHistory:+ def check_text_serialized(item: Any) -> bool:+ if isinstance(item, list):+ return any(check_text_serialized(i) for i in item)+ if isinstance(item, dict):+ return any(check_text_serialized(i) for i in item.values())+ if isinstance(item, str):+ return "CUSTOM-TYPE-START-IDENTIFIER" in item++ return False++ has_structured_images = False++ for call in history:+ if call.get("messages"):+ image_count = count_messages_with_image_url_pattern(call["messages"])+ if image_count > 0:+ has_structured_images = True++ break++ return ImagesInHistory(+ has_structured_images=has_structured_images,+ has_text_serialized_images=any(check_text_serialized(i) for i in history),+ )+++def test_reflection_lm_gets_structured_images():+ """+ Verify reflection LM receives structured image messages, not serialized text.+ """+ student = dspy.Predict("image: dspy.Image -> label: str")+ image = dspy.Image.from_url("https://example.com/test.jpg", download=False)+ example = dspy.Example(image=image, label="dog").with_inputs("image")++ reflection_lm = DummyLM(+ [+ {"improved_instruction": "Better instruction"},+ {"improved_instruction": "Enhanced visual analysis instruction"},+ {"improved_instruction": "Focus on key features"},+ {"improved_instruction": "Analyze visual patterns systematically"},+ {"improved_instruction": "Consider distinctive visual elements"},+ {"improved_instruction": "Enhance recognition accuracy"},+ {"improved_instruction": "Improve classification methodology"},+ ]+ )+ lm = DummyLM(+ [+ {"label": "cat"},+ {"label": "dog"},+ {"label": "animal"},+ {"label": "pet"},+ {"label": "feline"},+ {"label": "canine"},+ {"label": "mammal"},+ {"label": "creature"},+ {"label": "species"},+ {"label": "domestic"},+ {"label": "wild"},+ {"label": "carnivore"},+ {"label": "herbivore"},+ {"label": "quadruped"},+ {"label": "vertebrate"},+ ]+ )+ dspy.settings.configure(lm=lm)++ gepa = dspy.GEPA(+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.3,+ max_metric_calls=2,+ reflection_lm=reflection_lm,+ instruction_proposer=instruction_proposal.MultiModalInstructionProposer(),+ )++ gepa.compile(student, trainset=[example], valset=[example])++ assert len(lm.history) > 0, "LM should have been called"+ assert len(reflection_lm.history) > 0, "Reflection LM should have been called"++ images_in_history = check_images_in_history(reflection_lm.history)++ assert images_in_history.has_structured_images, "Reflection LM should have received structured images"+ assert not images_in_history.has_text_serialized_images, "Reflection LM received serialized images in prompts"+++def test_custom_proposer_without_reflection_lm():+ """Test that custom instruction proposers can work without reflection_lm when using updated GEPA core."""++ # External reflection LM managed by the custom proposer+ external_reflection_lm = DummyLM(+ [+ {"improved_instruction": "External LM response"},+ {"improved_instruction": "Enhanced instruction"},+ {"improved_instruction": "Better guidance"},+ {"improved_instruction": "Optimized instruction"},+ {"improved_instruction": "Refined approach"},+ ]+ )++ class ProposerWithExternalLM:+ def __call__(self, candidate, reflective_dataset, components_to_update):+ # This proposer manages its own external reflection LM+ with dspy.context(lm=external_reflection_lm):+ # Use external LM for reflection (optional - could be any custom logic)+ external_reflection_lm([{"role": "user", "content": "Improve this instruction"}])+ return {name: f"Externally-improved: {candidate[name]}" for name in components_to_update}++ student = dspy.Predict("text -> label")+ example = dspy.Example(text="test input", label="test").with_inputs("text")++ # Use a robust dummy LM with enough responses for optimization steps+ lm = DummyLM(+ [+ {"label": "test"},+ {"label": "result"},+ {"label": "output"},+ {"label": "response"},+ {"label": "classification"},+ {"label": "prediction"},+ {"label": "category"},+ {"label": "type"},+ {"label": "class"},+ {"label": "group"},+ {"label": "kind"},+ {"label": "variant"},+ {"label": "form"},+ {"label": "style"},+ {"label": "mode"},+ ]+ )+ dspy.settings.configure(lm=lm)++ # Test the full flexibility: no reflection_lm provided to GEPA at all!+ # The updated GEPA core library now allows this when using custom proposers+ gepa = dspy.GEPA(+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.7, # Score to trigger optimization+ max_metric_calls=5, # More calls to allow proper optimization+ reflection_lm=None, # No reflection_lm provided - this now works!+ instruction_proposer=ProposerWithExternalLM(),+ )++ result = gepa.compile(student, trainset=[example], valset=[example])++ assert result is not None+ assert len(lm.history) > 0, "Main LM should have been called"+ assert len(external_reflection_lm.history) > 0, "External reflection LM should have been called by custom proposer"+++def test_image_serialization_into_strings():+ """+ Test that demonstrates the image serialization problem when calling lm directly with serialized image data.+ """++ class InstructionProposerCallingLMDirectly:+ def __call__(+ self,+ candidate: dict[str, str],+ reflective_dataset: dict[str, list[dict[str, Any]]],+ components_to_update: list[str],+ ) -> dict[str, str]:+ updated_components = {}++ for component_name in components_to_update:+ if component_name not in candidate or component_name not in reflective_dataset:+ continue++ current_instruction = candidate[component_name]+ component_data = reflective_dataset[component_name]++ feedback_analysis = "Feedback analysis:\n"+ for i, example in enumerate(component_data):+ feedback_analysis += f"Example {i + 1}:\n"++ # Non ideal approach: extract and serialize image objects directly+ inputs = example.get("Inputs", {})+ for key, value in inputs.items():+ feedback_analysis += f" {key}: {value}\n"++ outputs = example.get("Generated Outputs", {})+ feedback = example.get("Feedback", "")+ feedback_analysis += f" Outputs: {outputs}\n"+ feedback_analysis += f" Feedback: {feedback}\n\n"++ context_lm = dspy.settings.lm+ messages = [+ {"role": "system", "content": "You are an instruction improvement assistant."},+ {+ "role": "user",+ "content": f"Current instruction: {current_instruction}\n\nFeedback: {feedback_analysis}\n\nProvide an improved instruction:",+ },+ ]++ result = context_lm(messages=messages)+ updated_components[component_name] = result[0]++ return updated_components++ direct_lm_call_proposer = InstructionProposerCallingLMDirectly()++ student = dspy.Predict("image -> label")++ image = dspy.Image.from_url("https://picsum.photos/id/237/200/300", download=False)++ examples = [+ dspy.Example(image=image, label="cat").with_inputs("image"),+ dspy.Example(image=image, label="animal").with_inputs("image"),+ ]++ lm = DummyLM(+ [+ {"label": "cat"},+ {"label": "dog"},+ {"label": "animal"},+ {"label": "pet"},+ {"label": "feline"},+ {"label": "mammal"},+ {"label": "creature"},+ {"label": "species"},+ {"label": "domestic"},+ {"label": "wild"},+ {"label": "carnivore"},+ {"label": "herbivore"},+ ]+ )+ dspy.settings.configure(lm=lm)++ reflection_lm = DummyLM(+ [+ {"improved_instruction": "Be more specific about image analysis"},+ {"improved_instruction": "Focus on visual features when classifying"},+ {"improved_instruction": "Consider contextual clues in the image"},+ {"improved_instruction": "Analyze shape, color, and texture patterns"},+ {"improved_instruction": "Look for distinguishing characteristics"},+ ]+ )++ gepa = dspy.GEPA(+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.3,+ max_metric_calls=5,+ reflection_lm=reflection_lm,+ instruction_proposer=direct_lm_call_proposer,+ )++ gepa.compile(student, trainset=examples, valset=examples)++ assert len(lm.history) > 0, "LM should have been called"+ assert len(reflection_lm.history) > 0, "Reflection LM should have been called"++ images_in_history = check_images_in_history(reflection_lm.history)++ assert images_in_history.has_text_serialized_images, (+ "Expected to find serialized images (CUSTOM-TYPE-START-IDENTIFIER)"+ )+++def test_default_proposer():+ student = dspy.Predict("image -> label")++ image = dspy.Image.from_url("https://picsum.photos/id/237/200/300", download=False)++ examples = [+ dspy.Example(image=image, label="cat").with_inputs("image"),+ dspy.Example(image=image, label="animal").with_inputs("image"),+ ]++ lm = DummyLM(+ [+ {"label": "cat"},+ {"label": "dog"},+ {"label": "animal"},+ {"label": "pet"},+ {"label": "feline"},+ {"label": "mammal"},+ {"label": "creature"},+ {"label": "species"},+ {"label": "domestic"},+ {"label": "wild"},+ {"label": "carnivore"},+ {"label": "herbivore"},+ ]+ )+ dspy.settings.configure(lm=lm)++ reflection_lm = DummyLM(+ [+ {"improved_instruction": "Be more specific about image analysis"},+ {"improved_instruction": "Focus on visual features when classifying"},+ {"improved_instruction": "Consider contextual clues in the image"},+ {"improved_instruction": "Analyze shape, color, and texture patterns"},+ {"improved_instruction": "Look for distinguishing characteristics"},+ ]+ )++ gepa = dspy.GEPA(+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.3,+ max_metric_calls=5,+ reflection_lm=reflection_lm,+ )++ gepa.compile(student, trainset=examples, valset=examples)++ assert len(lm.history) > 0, "LM should have been called"+ assert len(reflection_lm.history) > 0, "Reflection LM should have been called"++ images_in_history = check_images_in_history(reflection_lm.history)++ assert images_in_history.has_text_serialized_images, (+ "Expected to find serialized images (CUSTOM-TYPE-START-IDENTIFIER)"+ )diff --git a/uv.lock b/uv.lockindex 0c564c8de..9611681f6 100644--- a/uv.lock+++ b/uv.lock@@ -664,7 +664,7 @@ wheels = [[[package]]name = "dspy"-version = "3.0.0"+version = "3.0.3"source = { editable = "." }dependencies = [{ name = "anyio" },@@ -740,7 +740,7 @@ requires-dist = [{ name = "datamodel-code-generator", marker = "extra == 'dev'", specifier = ">=0.26.3" },{ name = "datasets", marker = "extra == 'test-extras'", specifier = ">=2.14.6" },{ name = "diskcache", specifier = ">=5.6.0" },- { name = "gepa", specifier = "==0.0.2" },+ { name = "gepa", extras = ["dspy"], specifier = "==0.0.8" },{ name = "joblib", specifier = "~=1.3" },{ name = "json-repair", specifier = ">=0.30.0" },{ name = "langchain-core", marker = "extra == 'langchain'" },@@ -956,11 +956,11 @@ wheels = [[[package]]name = "gepa"-version = "0.0.2"+version = "0.0.8"source = { registry = "https://pypi.org/simple" }-sdist = { url = "https://files.pythonhosted.org/packages/74/8b/2aba83c1a85d1260151c7b0ec57c4b34901ae7a74f27eba4e7532c2f80c8/gepa-0.0.2.tar.gz", hash = "sha256:0fa8ca333e4a69eec68087a68cdd651c9e4b8df0c147009f42a5563f1b9702dc", size = 32253, upload-time = "2025-08-12T02:26:49.112Z" }+sdist = { url = "https://files.pythonhosted.org/packages/0c/19/126e6e897595c0374379dc3d7d34cf60e941e47d278fad3e05b313fc7989/gepa-0.0.8.tar.gz", hash = "sha256:c786bd7eff5547d51930171fb6afb69d6f970807d4dde73f521da08eb36253fe", size = 57149, upload-time = "2025-09-03T01:55:55.724Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/58/9c/a1111dfc06e1f4b3adac51670b76b45b8385ed510f35bca6c002ca25eef8/gepa-0.0.2-py3-none-any.whl", hash = "sha256:9894ceefd53d46a6811cf8737b698c99f626777a773307dafc5f8e0038007e53", size = 34191, upload-time = "2025-08-12T02:26:47.625Z" },+ { url = "https://files.pythonhosted.org/packages/36/bd/9c3adfbe4a903a80c270e186aeca2989c4cf46fce11a872c583b0e5d3308/gepa-0.0.8-py3-none-any.whl", hash = "sha256:086b40ec8923692af3ff2d563c74f31ddd57dfe8443ce214cd4a3628382252c5", size = 62141, upload-time = "2025-09-03T01:55:54.451Z" },][[package]]