Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Add custom instruction_proposer support to GEPA with multimodal (dspy.Image) handling (#8737)

Andres Serrano91e7e23bf977
diff --git a/dspy/teleprompt/gepa/gepa.py b/dspy/teleprompt/gepa/gepa.py
index a4ea0878c..14aeac89e 100644
--- a/dspy/teleprompt/gepa/gepa.py
+++ b/dspy/teleprompt/gepa/gepa.py
@@ -2,17 +2,16 @@ import inspect
import logging
import random
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Literal, Optional, Protocol, Union
+from typing import Any, Literal, Optional, Protocol, Union
+
+from gepa import GEPAResult
+from gepa.core.adapter import ProposalFn
from dspy.clients.lm import LM
from dspy.primitives import Example, Module, Prediction
+from dspy.teleprompt.gepa.gepa_utils import DspyAdapter, DSPyTrace, PredictorFeedbackFn, ScoreWithFeedback
from dspy.teleprompt.teleprompt import Teleprompter
-if TYPE_CHECKING:
- from gepa import GEPAResult
-
- from dspy.teleprompt.gepa.gepa_utils import DspyAdapter, DSPyTrace, PredictorFeedbackFn, ScoreWithFeedback
-
logger = logging.getLogger(__name__)
AUTO_RUN_SETTINGS = {
@@ -210,6 +209,18 @@ class GEPA(Teleprompter):
a strong reflection model. Consider using `dspy.LM(model='gpt-5', temperature=1.0, max_tokens=32000)`
for optimal performance.
skip_perfect_score: Whether to skip examples with perfect scores during reflection. Default is True.
+ instruction_proposer: Optional custom instruction proposer implementing GEPA's ProposalFn protocol.
+ If provided, GEPA will use this custom proposer instead of its default instruction proposal
+ mechanism to generate improved instructions based on feedback from failed examples. This is
+ particularly useful when you need specialized instruction generation for multimodal inputs
+ (like dspy.Image) or custom types. Use `MultiModalInstructionProposer()` from
+ `dspy.teleprompt.gepa.instruction_proposal` for handling visual content. If None (default),
+ GEPA uses its built-in text-optimized proposer (see `gepa.strategies.instruction_proposal.InstructionProposalSignature`
+ for reference implementation).
+
+ Note: When both instruction_proposer and reflection_lm are set, the instruction_proposer is called
+ in the reflection_lm context. However, reflection_lm is optional when using a custom instruction_proposer.
+ Custom instruction proposers can invoke their own LLMs if needed.
add_format_failure_as_feedback: Whether to add format failures as feedback. Default is False.
use_merge: Whether to use merge-based optimization. Default is True.
max_merge_invocations: The maximum number of merge invocations to perform. Default is 5.
@@ -260,12 +271,13 @@ class GEPA(Teleprompter):
auto: Literal["light", "medium", "heavy"] | None = None,
max_full_evals: int | None = None,
max_metric_calls: int | None = None,
- # Reflection based configuration
+ # Reflection configuration
reflection_minibatch_size: int = 3,
candidate_selection_strategy: Literal["pareto", "current_best"] = "pareto",
reflection_lm: LM | None = None,
skip_perfect_score: bool = True,
add_format_failure_as_feedback: bool = False,
+ instruction_proposer: "ProposalFn | None" = None,
# Merge-based configuration
use_merge: bool = True,
max_merge_invocations: int | None = 5,
@@ -309,12 +321,17 @@ class GEPA(Teleprompter):
self.max_full_evals = max_full_evals
self.max_metric_calls = max_metric_calls
- # Reflection based configuration
+ # Reflection configuration
self.reflection_minibatch_size = reflection_minibatch_size
self.candidate_selection_strategy = candidate_selection_strategy
- # self.reflection_lm = reflection_lm
- assert reflection_lm is not None, "GEPA requires a reflection language model to be provided. Typically, you can use `dspy.LM(model='gpt-5', temperature=1.0, max_tokens=32000)` to get a good reflection model. Reflection LM is used by GEPA to reflect on the behavior of the program and propose new instructions, and will benefit from a strong model."
- self.reflection_lm = lambda x: reflection_lm(x)[0]
+
+ assert reflection_lm is not None or instruction_proposer is not None, (
+ "GEPA requires a reflection language model, or custom instruction proposer to be provided. "
+ "Typically, you can use `dspy.LM(model='gpt-5', temperature=1.0, max_tokens=32000)` to get a good reflection model. "
+ "Reflection LM is used by GEPA to reflect on the behavior of the program and propose new instructions, and will benefit from a strong model. "
+ )
+
+ self.reflection_lm = reflection_lm
self.skip_perfect_score = skip_perfect_score
self.add_format_failure_as_feedback = add_format_failure_as_feedback
@@ -341,6 +358,8 @@ class GEPA(Teleprompter):
# Reproducibility
self.seed = seed
+ self.custom_instruction_proposer = instruction_proposer
+
def auto_budget(self, num_preds, num_candidates, valset_size: int, minibatch_size: int = 35, full_eval_steps: int = 5) -> int:
import numpy as np
num_trials = int(max(2 * (num_preds * 2) * np.log2(num_candidates), 1.5 * num_candidates))
@@ -452,10 +471,10 @@ class GEPA(Teleprompter):
num_threads=self.num_threads,
add_format_failure_as_feedback=self.add_format_failure_as_feedback,
rng=rng,
+ reflection_lm=self.reflection_lm,
+ custom_instruction_proposer=self.custom_instruction_proposer,
)
- reflection_lm = self.reflection_lm
-
# Instantiate GEPA with the simpler adapter-based API
base_program = {name: pred.signature.instructions for name, pred in student.named_predictors()}
gepa_result: GEPAResult = optimize(
@@ -465,7 +484,7 @@ class GEPA(Teleprompter):
adapter=adapter,
# Reflection-based configuration
- reflection_lm=reflection_lm,
+ reflection_lm=(lambda x: self.reflection_lm(x)[0]) if self.reflection_lm is not None else None,
candidate_selection_strategy=self.candidate_selection_strategy,
skip_perfect_score=self.skip_perfect_score,
reflection_minibatch_size=self.reflection_minibatch_size,
diff --git a/dspy/teleprompt/gepa/gepa_utils.py b/dspy/teleprompt/gepa/gepa_utils.py
index 95d1fc17f..66b3b4636 100644
--- a/dspy/teleprompt/gepa/gepa_utils.py
+++ b/dspy/teleprompt/gepa/gepa_utils.py
@@ -3,9 +3,12 @@ import random
from typing import Any, Callable, Protocol
from gepa import EvaluationBatch, GEPAAdapter
+from gepa.core.adapter import ProposalFn
+import dspy
from dspy.adapters.chat_adapter import ChatAdapter
from dspy.adapters.types import History
+from dspy.adapters.types.base_type import Type
from dspy.evaluate import Evaluate
from dspy.primitives import Example, Prediction
from dspy.teleprompt.bootstrap_trace import TraceData
@@ -58,6 +61,8 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
num_threads: int | None = None,
add_format_failure_as_feedback: bool = False,
rng: random.Random | None = None,
+ reflection_lm=None,
+ custom_instruction_proposer: "ProposalFn | None" = None,
):
self.student = student_module
self.metric_fn = metric_fn
@@ -66,6 +71,34 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
self.num_threads = num_threads
self.add_format_failure_as_feedback = add_format_failure_as_feedback
self.rng = rng or random.Random(0)
+ self.reflection_lm = reflection_lm
+ self.custom_instruction_proposer = custom_instruction_proposer
+
+ if self.custom_instruction_proposer is not None:
+ # We are only overriding the propose_new_texts method when a custom
+ # instruction proposer is provided. Otherwise, we use the GEPA
+ # default propose_new_texts.
+
+ def custom_propose_new_texts(
+ candidate: dict[str, str],
+ reflective_dataset: dict[str, list[dict[str, Any]]],
+ components_to_update: list[str]
+ ) -> dict[str, str]:
+ if self.reflection_lm is not None:
+ with dspy.context(lm=self.reflection_lm):
+ return self.custom_instruction_proposer(
+ candidate=candidate,
+ reflective_dataset=reflective_dataset,
+ components_to_update=components_to_update
+ )
+ else:
+ return self.custom_instruction_proposer(
+ candidate=candidate,
+ reflective_dataset=reflective_dataset,
+ components_to_update=components_to_update
+ )
+
+ self.propose_new_texts = custom_propose_new_texts
# Cache predictor names/signatures
self.named_predictors = list(self.student.named_predictors())
@@ -184,7 +217,12 @@ class DspyAdapter(GEPAAdapter[Example, TraceData, Prediction]):
for input_key, input_val in inputs.items():
if contains_history and input_key == history_key_name:
continue
- new_inputs[input_key] = str(input_val)
+
+ if isinstance(input_val, Type) and self.custom_instruction_proposer is not None:
+ # Keep original object - will be properly formatted when sent to reflection LM
+ new_inputs[input_key] = input_val
+ else:
+ new_inputs[input_key] = str(input_val)
if isinstance(outputs, FailedPrediction):
s = "Couldn't parse the output as per the expected output format. The model's raw response was:\n"
diff --git a/dspy/teleprompt/gepa/instruction_proposal.py b/dspy/teleprompt/gepa/instruction_proposal.py
new file mode 100644
index 000000000..23b7ab180
--- /dev/null
+++ b/dspy/teleprompt/gepa/instruction_proposal.py
@@ -0,0 +1,311 @@
+from typing import Any
+
+from gepa.core.adapter import ProposalFn
+
+import dspy
+from dspy.adapters.types.base_type import Type
+
+
+class GenerateEnhancedMultimodalInstructionFromFeedback(dspy.Signature):
+ """I provided an assistant with instructions to perform a task involving visual content, but the assistant's performance needs improvement based on the examples and feedback below.
+
+ Your task is to write a better instruction for the assistant that addresses the specific issues identified in the feedback, with particular attention to how visual and textual information should be analyzed and integrated.
+
+ ## Analysis Steps:
+ 1. **Read the inputs carefully** and identify both the visual and textual input formats, understanding how they work together
+ 2. **Read all the assistant responses and corresponding feedback** to understand what went wrong with visual analysis, text processing, or their integration
+ 3. **Identify visual analysis patterns** - what visual features, relationships, or details are important for this task
+ 4. **Identify domain-specific knowledge** about both visual and textual aspects, as this information may not be available to the assistant in the future
+ 5. **Look for successful visual-textual integration strategies** and include these patterns in the instruction
+ 6. **Address specific visual analysis issues** mentioned in the feedback
+
+ ## Instruction Requirements:
+ - **Clear task definition** explaining how to process both visual and textual inputs
+ - **Visual analysis guidance** specific to this task (what to look for, how to describe, what features matter)
+ - **Integration strategies** for combining visual observations with textual information
+ - **Domain-specific knowledge** about visual concepts, terminology, or relationships
+ - **Error prevention guidance** for common visual analysis mistakes shown in the feedback
+ - **Precise, actionable language** for both visual and textual processing
+
+ Focus on creating an instruction that helps the assistant properly analyze visual content, integrate it with textual information, and avoid the specific visual analysis mistakes shown in the examples."""
+
+ current_instruction = dspy.InputField(
+ desc="The current instruction that was provided to the assistant to perform the multimodal task"
+ )
+ examples_with_feedback = dspy.InputField(
+ desc="Task examples with visual content showing inputs, assistant outputs, and feedback. "
+ "Pay special attention to feedback about visual analysis accuracy, visual-textual integration, "
+ "and any domain-specific visual knowledge that the assistant missed."
+ )
+
+ improved_instruction = dspy.OutputField(
+ desc="A better instruction for the assistant that addresses visual analysis issues, provides "
+ "clear guidance on how to process and integrate visual and textual information, includes "
+ "necessary visual domain knowledge, and prevents the visual analysis mistakes shown in the examples."
+ )
+
+
+class SingleComponentMultiModalProposer(dspy.Module):
+ """
+ dspy.Module for proposing improved instructions based on feedback.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.propose_instruction = dspy.Predict(GenerateEnhancedMultimodalInstructionFromFeedback)
+
+ def forward(self, current_instruction: str, reflective_dataset: list[dict[str, Any]]) -> str:
+ """
+ Generate an improved instruction based on current instruction and feedback examples.
+
+ Args:
+ current_instruction: The current instruction that needs improvement
+ reflective_dataset: List of examples with inputs, outputs, and feedback
+ May contain dspy.Image objects in inputs
+
+ Returns:
+ str: Improved instruction text
+ """
+ # Format examples with enhanced pattern recognition
+ formatted_examples, image_map = self._format_examples_with_pattern_analysis(reflective_dataset)
+
+ # Build kwargs for the prediction call
+ predict_kwargs = {
+ "current_instruction": current_instruction,
+ "examples_with_feedback": formatted_examples,
+ }
+
+ # Create a rich multimodal examples_with_feedback that includes both text and images
+ predict_kwargs["examples_with_feedback"] = self._create_multimodal_examples(formatted_examples, image_map)
+
+ # Use current dspy LM settings (GEPA will pass reflection_lm via context)
+ result = self.propose_instruction(**predict_kwargs)
+
+ return result.improved_instruction
+
+ def _format_examples_with_pattern_analysis(
+ self, reflective_dataset: list[dict[str, Any]]
+ ) -> tuple[str, dict[int, list[Type]]]:
+ """
+ Format examples with pattern analysis and feedback categorization.
+
+ Returns:
+ tuple: (formatted_text_with_patterns, image_map)
+ """
+ # First, use the existing proven formatting approach
+ formatted_examples, image_map = self._format_examples_for_instruction_generation(reflective_dataset)
+
+ # Enhanced analysis: categorize feedback patterns
+ feedback_analysis = self._analyze_feedback_patterns(reflective_dataset)
+
+ # Add pattern analysis to the formatted examples
+ if feedback_analysis["summary"]:
+ pattern_summary = self._create_pattern_summary(feedback_analysis)
+ enhanced_examples = f"{pattern_summary}\n\n{formatted_examples}"
+ return enhanced_examples, image_map
+
+ return formatted_examples, image_map
+
+ def _analyze_feedback_patterns(self, reflective_dataset: list[dict[str, Any]]) -> dict[str, Any]:
+ """
+ Analyze feedback patterns to provide better context for instruction generation.
+
+ Categorizes feedback into:
+ - Error patterns: Common mistakes and their types
+ - Success patterns: What worked well and should be preserved/emphasized
+ - Domain knowledge gaps: Missing information that should be included
+ - Task-specific guidance: Specific requirements or edge cases
+ """
+ analysis = {
+ "error_patterns": [],
+ "success_patterns": [],
+ "domain_knowledge_gaps": [],
+ "task_specific_guidance": [],
+ "summary": "",
+ }
+
+ # Simple pattern recognition - could be enhanced further
+ for example in reflective_dataset:
+ feedback = example.get("Feedback", "").lower()
+
+ # Identify error patterns
+ if any(error_word in feedback for error_word in ["incorrect", "wrong", "error", "failed", "missing"]):
+ analysis["error_patterns"].append(feedback)
+
+ # Identify success patterns
+ if any(
+ success_word in feedback for success_word in ["correct", "good", "accurate", "well", "successfully"]
+ ):
+ analysis["success_patterns"].append(feedback)
+
+ # Identify domain knowledge needs
+ if any(
+ knowledge_word in feedback
+ for knowledge_word in ["should know", "domain", "specific", "context", "background"]
+ ):
+ analysis["domain_knowledge_gaps"].append(feedback)
+
+ # Create summary if patterns were found
+ if any(analysis[key] for key in ["error_patterns", "success_patterns", "domain_knowledge_gaps"]):
+ analysis["summary"] = (
+ f"Patterns identified: {len(analysis['error_patterns'])} error(s), {len(analysis['success_patterns'])} success(es), {len(analysis['domain_knowledge_gaps'])} knowledge gap(s)"
+ )
+
+ return analysis
+
+ def _create_pattern_summary(self, feedback_analysis: dict[str, Any]) -> str:
+ """Create a summary of feedback patterns to help guide instruction generation."""
+
+ summary_parts = ["## Feedback Pattern Analysis\n"]
+
+ if feedback_analysis["error_patterns"]:
+ summary_parts.append(f"**Common Issues Found ({len(feedback_analysis['error_patterns'])} examples):**")
+ summary_parts.append("Focus on preventing these types of mistakes in the new instruction.\n")
+
+ if feedback_analysis["success_patterns"]:
+ summary_parts.append(
+ f"**Successful Approaches Found ({len(feedback_analysis['success_patterns'])} examples):**"
+ )
+ summary_parts.append("Build on these successful strategies in the new instruction.\n")
+
+ if feedback_analysis["domain_knowledge_gaps"]:
+ summary_parts.append(
+ f"**Domain Knowledge Needs Identified ({len(feedback_analysis['domain_knowledge_gaps'])} examples):**"
+ )
+ summary_parts.append("Include this specialized knowledge in the new instruction.\n")
+
+ return "\n".join(summary_parts)
+
+ def _format_examples_for_instruction_generation(
+ self, reflective_dataset: list[dict[str, Any]]
+ ) -> tuple[str, dict[int, list[Type]]]:
+ """
+ Format examples using GEPA's markdown structure while preserving image objects.
+
+ Returns:
+ tuple: (formatted_text, image_map) where image_map maps example_index -> list[images]
+ """
+
+ def render_value_with_images(value, level=3, example_images=None):
+ if example_images is None:
+ example_images = []
+
+ if isinstance(value, Type):
+ image_idx = len(example_images) + 1
+ example_images.append(value)
+ return f"[IMAGE-{image_idx} - see visual content]\n\n"
+ elif isinstance(value, dict):
+ s = ""
+ for k, v in value.items():
+ s += f"{'#' * level} {k}\n"
+ s += render_value_with_images(v, min(level + 1, 6), example_images)
+ if not value:
+ s += "\n"
+ return s
+ elif isinstance(value, (list, tuple)):
+ s = ""
+ for i, item in enumerate(value):
+ s += f"{'#' * level} Item {i + 1}\n"
+ s += render_value_with_images(item, min(level + 1, 6), example_images)
+ if not value:
+ s += "\n"
+ return s
+ else:
+ return f"{str(value).strip()}\n\n"
+
+ def convert_sample_to_markdown_with_images(sample, example_num):
+ example_images = []
+ s = f"# Example {example_num}\n"
+
+ for key, val in sample.items():
+ s += f"## {key}\n"
+ s += render_value_with_images(val, level=3, example_images=example_images)
+
+ return s, example_images
+
+ formatted_parts = []
+ image_map = {}
+
+ for i, example_data in enumerate(reflective_dataset):
+ formatted_example, example_images = convert_sample_to_markdown_with_images(example_data, i + 1)
+ formatted_parts.append(formatted_example)
+
+ if example_images:
+ image_map[i] = example_images
+
+ formatted_text = "\n\n".join(formatted_parts)
+
+ if image_map:
+ total_images = sum(len(imgs) for imgs in image_map.values())
+ formatted_text = (
+ f"The examples below include visual content ({total_images} images total). "
+ "Please analyze both the text and visual elements when suggesting improvements.\n\n" + formatted_text
+ )
+
+ return formatted_text, image_map
+
+ def _create_multimodal_examples(self, formatted_text: str, image_map: dict[int, list[Type]]) -> Any:
+ """
+ Create a multimodal input that contains both text and images for the reflection LM.
+
+ Args:
+ formatted_text: The formatted text with image placeholders
+ image_map: Dictionary mapping example_index -> list[images] for structured access
+ """
+ if not image_map:
+ return formatted_text
+
+ # Collect all images from all examples
+ all_images = []
+ for example_images in image_map.values():
+ all_images.extend(example_images)
+
+ multimodal_content = [formatted_text]
+ multimodal_content.extend(all_images)
+ return multimodal_content
+
+
+class MultiModalInstructionProposer(ProposalFn):
+ """GEPA-compatible multimodal instruction proposer.
+
+ This class handles multimodal inputs (like dspy.Image) during GEPA optimization by using
+ a single-component proposer for each component that needs to be updated.
+ """
+
+ def __init__(self):
+ self.single_proposer = SingleComponentMultiModalProposer()
+
+ def __call__(
+ self,
+ candidate: dict[str, str],
+ reflective_dataset: dict[str, list[dict[str, Any]]],
+ components_to_update: list[str],
+ ) -> dict[str, str]:
+ """GEPA-compatible proposal function.
+
+ Args:
+ candidate: Current component name -> instruction mapping
+ reflective_dataset: Component name -> list of reflective examples
+ components_to_update: List of component names to update
+
+ Returns:
+ dict: Component name -> new instruction mapping
+ """
+ updated_components = {}
+
+ for component_name in components_to_update:
+ if component_name in candidate and component_name in reflective_dataset:
+ current_instruction = candidate[component_name]
+ component_reflective_data = reflective_dataset[component_name]
+
+ # Call the single-instruction proposer.
+ #
+ # In the future, proposals could consider multiple components instructions,
+ # instead of just the current instruction, for more holistic instruction proposals.
+ new_instruction = self.single_proposer(
+ current_instruction=current_instruction, reflective_dataset=component_reflective_data
+ )
+
+ updated_components[component_name] = new_instruction
+
+ return updated_components
diff --git a/pyproject.toml b/pyproject.toml
index 0babb99b5..7dabe4d48 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ dependencies = [
"rich>=13.7.1",
"numpy>=1.26.0",
"xxhash>=3.5.0",
- "gepa[dspy]==0.0.7"
+ "gepa[dspy]==0.0.8",
]
[project.optional-dependencies]
diff --git a/tests/teleprompt/test_gepa_instruction_proposer.py b/tests/teleprompt/test_gepa_instruction_proposer.py
new file mode 100644
index 000000000..358e6d90f
--- /dev/null
+++ b/tests/teleprompt/test_gepa_instruction_proposer.py
@@ -0,0 +1,353 @@
+from dataclasses import dataclass
+from typing import Any
+
+import dspy
+from dspy.teleprompt.gepa import instruction_proposal
+from dspy.utils.dummies import DummyLM
+
+
+def count_messages_with_image_url_pattern(messages):
+ """Helper to count image URLs in messages - borrowed from image adapter tests"""
+ pattern = {"type": "image_url", "image_url": {"url": lambda x: isinstance(x, str)}}
+
+ try:
+
+ def check_pattern(obj, pattern):
+ if isinstance(pattern, dict):
+ if not isinstance(obj, dict):
+ return False
+ return all(k in obj and check_pattern(obj[k], v) for k, v in pattern.items())
+ if callable(pattern):
+ return pattern(obj)
+ return obj == pattern
+
+ def count_patterns(obj, pattern):
+ count = 0
+ if check_pattern(obj, pattern):
+ count += 1
+ if isinstance(obj, dict):
+ count += sum(count_patterns(v, pattern) for v in obj.values())
+ if isinstance(obj, (list, tuple)):
+ count += sum(count_patterns(v, pattern) for v in obj)
+ return count
+
+ return count_patterns(messages, pattern)
+ except Exception:
+ return 0
+
+
+@dataclass
+class ImagesInHistory:
+ has_structured_images: bool
+ has_text_serialized_images: bool
+
+
+def check_images_in_history(history: list[Any]) -> ImagesInHistory:
+ def check_text_serialized(item: Any) -> bool:
+ if isinstance(item, list):
+ return any(check_text_serialized(i) for i in item)
+ if isinstance(item, dict):
+ return any(check_text_serialized(i) for i in item.values())
+ if isinstance(item, str):
+ return "CUSTOM-TYPE-START-IDENTIFIER" in item
+
+ return False
+
+ has_structured_images = False
+
+ for call in history:
+ if call.get("messages"):
+ image_count = count_messages_with_image_url_pattern(call["messages"])
+ if image_count > 0:
+ has_structured_images = True
+
+ break
+
+ return ImagesInHistory(
+ has_structured_images=has_structured_images,
+ has_text_serialized_images=any(check_text_serialized(i) for i in history),
+ )
+
+
+def test_reflection_lm_gets_structured_images():
+ """
+ Verify reflection LM receives structured image messages, not serialized text.
+ """
+ student = dspy.Predict("image: dspy.Image -> label: str")
+ image = dspy.Image.from_url("https://example.com/test.jpg", download=False)
+ example = dspy.Example(image=image, label="dog").with_inputs("image")
+
+ reflection_lm = DummyLM(
+ [
+ {"improved_instruction": "Better instruction"},
+ {"improved_instruction": "Enhanced visual analysis instruction"},
+ {"improved_instruction": "Focus on key features"},
+ {"improved_instruction": "Analyze visual patterns systematically"},
+ {"improved_instruction": "Consider distinctive visual elements"},
+ {"improved_instruction": "Enhance recognition accuracy"},
+ {"improved_instruction": "Improve classification methodology"},
+ ]
+ )
+ lm = DummyLM(
+ [
+ {"label": "cat"},
+ {"label": "dog"},
+ {"label": "animal"},
+ {"label": "pet"},
+ {"label": "feline"},
+ {"label": "canine"},
+ {"label": "mammal"},
+ {"label": "creature"},
+ {"label": "species"},
+ {"label": "domestic"},
+ {"label": "wild"},
+ {"label": "carnivore"},
+ {"label": "herbivore"},
+ {"label": "quadruped"},
+ {"label": "vertebrate"},
+ ]
+ )
+ dspy.settings.configure(lm=lm)
+
+ gepa = dspy.GEPA(
+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.3,
+ max_metric_calls=2,
+ reflection_lm=reflection_lm,
+ instruction_proposer=instruction_proposal.MultiModalInstructionProposer(),
+ )
+
+ gepa.compile(student, trainset=[example], valset=[example])
+
+ assert len(lm.history) > 0, "LM should have been called"
+ assert len(reflection_lm.history) > 0, "Reflection LM should have been called"
+
+ images_in_history = check_images_in_history(reflection_lm.history)
+
+ assert images_in_history.has_structured_images, "Reflection LM should have received structured images"
+ assert not images_in_history.has_text_serialized_images, "Reflection LM received serialized images in prompts"
+
+
+def test_custom_proposer_without_reflection_lm():
+ """Test that custom instruction proposers can work without reflection_lm when using updated GEPA core."""
+
+ # External reflection LM managed by the custom proposer
+ external_reflection_lm = DummyLM(
+ [
+ {"improved_instruction": "External LM response"},
+ {"improved_instruction": "Enhanced instruction"},
+ {"improved_instruction": "Better guidance"},
+ {"improved_instruction": "Optimized instruction"},
+ {"improved_instruction": "Refined approach"},
+ ]
+ )
+
+ class ProposerWithExternalLM:
+ def __call__(self, candidate, reflective_dataset, components_to_update):
+ # This proposer manages its own external reflection LM
+ with dspy.context(lm=external_reflection_lm):
+ # Use external LM for reflection (optional - could be any custom logic)
+ external_reflection_lm([{"role": "user", "content": "Improve this instruction"}])
+ return {name: f"Externally-improved: {candidate[name]}" for name in components_to_update}
+
+ student = dspy.Predict("text -> label")
+ example = dspy.Example(text="test input", label="test").with_inputs("text")
+
+ # Use a robust dummy LM with enough responses for optimization steps
+ lm = DummyLM(
+ [
+ {"label": "test"},
+ {"label": "result"},
+ {"label": "output"},
+ {"label": "response"},
+ {"label": "classification"},
+ {"label": "prediction"},
+ {"label": "category"},
+ {"label": "type"},
+ {"label": "class"},
+ {"label": "group"},
+ {"label": "kind"},
+ {"label": "variant"},
+ {"label": "form"},
+ {"label": "style"},
+ {"label": "mode"},
+ ]
+ )
+ dspy.settings.configure(lm=lm)
+
+ # Test the full flexibility: no reflection_lm provided to GEPA at all!
+ # The updated GEPA core library now allows this when using custom proposers
+ gepa = dspy.GEPA(
+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.7, # Score to trigger optimization
+ max_metric_calls=5, # More calls to allow proper optimization
+ reflection_lm=None, # No reflection_lm provided - this now works!
+ instruction_proposer=ProposerWithExternalLM(),
+ )
+
+ result = gepa.compile(student, trainset=[example], valset=[example])
+
+ assert result is not None
+ assert len(lm.history) > 0, "Main LM should have been called"
+ assert len(external_reflection_lm.history) > 0, "External reflection LM should have been called by custom proposer"
+
+
+def test_image_serialization_into_strings():
+ """
+ Test that demonstrates the image serialization problem when calling lm directly with serialized image data.
+ """
+
+ class InstructionProposerCallingLMDirectly:
+ def __call__(
+ self,
+ candidate: dict[str, str],
+ reflective_dataset: dict[str, list[dict[str, Any]]],
+ components_to_update: list[str],
+ ) -> dict[str, str]:
+ updated_components = {}
+
+ for component_name in components_to_update:
+ if component_name not in candidate or component_name not in reflective_dataset:
+ continue
+
+ current_instruction = candidate[component_name]
+ component_data = reflective_dataset[component_name]
+
+ feedback_analysis = "Feedback analysis:\n"
+ for i, example in enumerate(component_data):
+ feedback_analysis += f"Example {i + 1}:\n"
+
+ # Non ideal approach: extract and serialize image objects directly
+ inputs = example.get("Inputs", {})
+ for key, value in inputs.items():
+ feedback_analysis += f" {key}: {value}\n"
+
+ outputs = example.get("Generated Outputs", {})
+ feedback = example.get("Feedback", "")
+ feedback_analysis += f" Outputs: {outputs}\n"
+ feedback_analysis += f" Feedback: {feedback}\n\n"
+
+ context_lm = dspy.settings.lm
+ messages = [
+ {"role": "system", "content": "You are an instruction improvement assistant."},
+ {
+ "role": "user",
+ "content": f"Current instruction: {current_instruction}\n\nFeedback: {feedback_analysis}\n\nProvide an improved instruction:",
+ },
+ ]
+
+ result = context_lm(messages=messages)
+ updated_components[component_name] = result[0]
+
+ return updated_components
+
+ direct_lm_call_proposer = InstructionProposerCallingLMDirectly()
+
+ student = dspy.Predict("image -> label")
+
+ image = dspy.Image.from_url("https://picsum.photos/id/237/200/300", download=False)
+
+ examples = [
+ dspy.Example(image=image, label="cat").with_inputs("image"),
+ dspy.Example(image=image, label="animal").with_inputs("image"),
+ ]
+
+ lm = DummyLM(
+ [
+ {"label": "cat"},
+ {"label": "dog"},
+ {"label": "animal"},
+ {"label": "pet"},
+ {"label": "feline"},
+ {"label": "mammal"},
+ {"label": "creature"},
+ {"label": "species"},
+ {"label": "domestic"},
+ {"label": "wild"},
+ {"label": "carnivore"},
+ {"label": "herbivore"},
+ ]
+ )
+ dspy.settings.configure(lm=lm)
+
+ reflection_lm = DummyLM(
+ [
+ {"improved_instruction": "Be more specific about image analysis"},
+ {"improved_instruction": "Focus on visual features when classifying"},
+ {"improved_instruction": "Consider contextual clues in the image"},
+ {"improved_instruction": "Analyze shape, color, and texture patterns"},
+ {"improved_instruction": "Look for distinguishing characteristics"},
+ ]
+ )
+
+ gepa = dspy.GEPA(
+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.3,
+ max_metric_calls=5,
+ reflection_lm=reflection_lm,
+ instruction_proposer=direct_lm_call_proposer,
+ )
+
+ gepa.compile(student, trainset=examples, valset=examples)
+
+ assert len(lm.history) > 0, "LM should have been called"
+ assert len(reflection_lm.history) > 0, "Reflection LM should have been called"
+
+ images_in_history = check_images_in_history(reflection_lm.history)
+
+ assert images_in_history.has_text_serialized_images, (
+ "Expected to find serialized images (CUSTOM-TYPE-START-IDENTIFIER)"
+ )
+
+
+def test_default_proposer():
+ student = dspy.Predict("image -> label")
+
+ image = dspy.Image.from_url("https://picsum.photos/id/237/200/300", download=False)
+
+ examples = [
+ dspy.Example(image=image, label="cat").with_inputs("image"),
+ dspy.Example(image=image, label="animal").with_inputs("image"),
+ ]
+
+ lm = DummyLM(
+ [
+ {"label": "cat"},
+ {"label": "dog"},
+ {"label": "animal"},
+ {"label": "pet"},
+ {"label": "feline"},
+ {"label": "mammal"},
+ {"label": "creature"},
+ {"label": "species"},
+ {"label": "domestic"},
+ {"label": "wild"},
+ {"label": "carnivore"},
+ {"label": "herbivore"},
+ ]
+ )
+ dspy.settings.configure(lm=lm)
+
+ reflection_lm = DummyLM(
+ [
+ {"improved_instruction": "Be more specific about image analysis"},
+ {"improved_instruction": "Focus on visual features when classifying"},
+ {"improved_instruction": "Consider contextual clues in the image"},
+ {"improved_instruction": "Analyze shape, color, and texture patterns"},
+ {"improved_instruction": "Look for distinguishing characteristics"},
+ ]
+ )
+
+ gepa = dspy.GEPA(
+ metric=lambda gold, pred, trace=None, pred_name=None, pred_trace=None: 0.3,
+ max_metric_calls=5,
+ reflection_lm=reflection_lm,
+ )
+
+ gepa.compile(student, trainset=examples, valset=examples)
+
+ assert len(lm.history) > 0, "LM should have been called"
+ assert len(reflection_lm.history) > 0, "Reflection LM should have been called"
+
+ images_in_history = check_images_in_history(reflection_lm.history)
+
+ assert images_in_history.has_text_serialized_images, (
+ "Expected to find serialized images (CUSTOM-TYPE-START-IDENTIFIER)"
+ )
diff --git a/uv.lock b/uv.lock
index 0c564c8de..9611681f6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -664,7 +664,7 @@ wheels = [
[[package]]
name = "dspy"
-version = "3.0.0"
+version = "3.0.3"
source = { editable = "." }
dependencies = [
{ name = "anyio" },
@@ -740,7 +740,7 @@ requires-dist = [
{ name = "datamodel-code-generator", marker = "extra == 'dev'", specifier = ">=0.26.3" },
{ name = "datasets", marker = "extra == 'test-extras'", specifier = ">=2.14.6" },
{ name = "diskcache", specifier = ">=5.6.0" },
- { name = "gepa", specifier = "==0.0.2" },
+ { name = "gepa", extras = ["dspy"], specifier = "==0.0.8" },
{ name = "joblib", specifier = "~=1.3" },
{ name = "json-repair", specifier = ">=0.30.0" },
{ name = "langchain-core", marker = "extra == 'langchain'" },
@@ -956,11 +956,11 @@ wheels = [
[[package]]
name = "gepa"
-version = "0.0.2"
+version = "0.0.8"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/74/8b/2aba83c1a85d1260151c7b0ec57c4b34901ae7a74f27eba4e7532c2f80c8/gepa-0.0.2.tar.gz", hash = "sha256:0fa8ca333e4a69eec68087a68cdd651c9e4b8df0c147009f42a5563f1b9702dc", size = 32253, upload-time = "2025-08-12T02:26:49.112Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/0c/19/126e6e897595c0374379dc3d7d34cf60e941e47d278fad3e05b313fc7989/gepa-0.0.8.tar.gz", hash = "sha256:c786bd7eff5547d51930171fb6afb69d6f970807d4dde73f521da08eb36253fe", size = 57149, upload-time = "2025-09-03T01:55:55.724Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/58/9c/a1111dfc06e1f4b3adac51670b76b45b8385ed510f35bca6c002ca25eef8/gepa-0.0.2-py3-none-any.whl", hash = "sha256:9894ceefd53d46a6811cf8737b698c99f626777a773307dafc5f8e0038007e53", size = 34191, upload-time = "2025-08-12T02:26:47.625Z" },
+ { url = "https://files.pythonhosted.org/packages/36/bd/9c3adfbe4a903a80c270e186aeca2989c4cf46fce11a872c583b0e5d3308/gepa-0.0.8-py3-none-any.whl", hash = "sha256:086b40ec8923692af3ff2d563c74f31ddd57dfe8443ce214cd4a3628382252c5", size = 62141, upload-time = "2025-09-03T01:55:54.451Z" },
]
[[package]]