Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

Push updated comparator code

Herumb Shandilyaff6bad922053
diff --git a/dspy/predict/avatar/avatar.py b/dspy/predict/avatar/avatar.py
index dd38a4485..546bcefdc 100644
--- a/dspy/predict/avatar/avatar.py
+++ b/dspy/predict/avatar/avatar.py
@@ -4,8 +4,8 @@ from copy import deepcopy
from pydantic.fields import FieldInfo
from dspy.predict.avatar.signatures import Actor
-from dspy.predict.avatar.models import Action, Tool
from dspy.signatures.signature import ensure_signature
+from dspy.predict.avatar.models import Action, ActionOutput, Tool
def get_number_with_suffix(number: int) -> str:
@@ -130,7 +130,8 @@ class Avatar(dspy.Module):
def forward(self, **kwargs):
- print("Starting the task...")
+ if self.verbose:
+ print("Starting the task...")
args = {
"goal" : self.signature.__doc__,
@@ -146,6 +147,7 @@ class Avatar(dspy.Module):
idx = 1
tool_name = None
+ action_results: list[ActionOutput] = []
max_iters = None if "max_iters" not in kwargs else kwargs["max_iters"]
while tool_name != "Finish" and (max_iters > 0 if max_iters else True):
@@ -160,6 +162,14 @@ class Avatar(dspy.Module):
if tool_name != "Finish":
tool_output = self._call_tool(tool_name, tool_input_query)
+ action_results.append(
+ ActionOutput(
+ tool_name=tool_name,
+ tool_input_query=tool_input_query,
+ tool_output=tool_output
+ )
+ )
+
self._update_signature(idx)
args[f"action_{idx}"] = action
@@ -177,8 +187,9 @@ class Avatar(dspy.Module):
max_iters -= 1
final_answer = self.actor(**args)
- self.actor = self.actor_clone
+ self.actor = deepcopy(self.actor_clone)
return dspy.Prediction(
- **{key: getattr(final_answer, key) for key in self.output_fields.keys()}
+ **{key: getattr(final_answer, key) for key in self.output_fields.keys()},
+ actions=action_results,
)
diff --git a/dspy/predict/avatar/models.py b/dspy/predict/avatar/models.py
index f26642a9d..63b43f11c 100644
--- a/dspy/predict/avatar/models.py
+++ b/dspy/predict/avatar/models.py
@@ -6,9 +6,10 @@ class Tool(BaseModel):
tool: Any
name: str
desc: Optional[str]
+ input_type: Optional[str] = None
def __str__(self) -> str:
- return f"{self.name}: {self.desc}"
+ return f"{self.name}{f'(valid_input: {self.input_type})' if self.input_type else ''}: {self.desc}"
def __repr__(self) -> str:
return self.__str__()
@@ -17,3 +18,9 @@ class Tool(BaseModel):
class Action(BaseModel):
tool_name: Any = Field(..., description="Name of the tool to use.")
tool_input_query: Any = Field(..., description="Query to pass as input to the tool.")
+
+
+class ActionOutput(BaseModel):
+ tool_name: str
+ tool_input_query: str
+ tool_output: str
\ No newline at end of file
diff --git a/dspy/predict/avatar/signatures.py b/dspy/predict/avatar/signatures.py
index e47dd2a3c..20b3a4a5e 100644
--- a/dspy/predict/avatar/signatures.py
+++ b/dspy/predict/avatar/signatures.py
@@ -5,7 +5,9 @@ from dspy.predict.avatar.models import Action
class Actor(dspy.Signature):
"""You will be given `Tools` which will be a list of tools to use to accomplish the `Goal`. Given the user query, your task is to decide which tool to use and what input values to provide.
-You will output action needed to accomplish the `Goal`. `Action` should have a tool to use and the input query to pass to the tool."""
+You will output action needed to accomplish the `Goal`. `Action` should have a tool to use and the input query to pass to the tool.
+
+Note: You can opt to use no tools and provide the final answer directly. You can also one tool multiple times with different input queries if applicable."""
goal: str = dspy.InputField(
prefix="Goal:",
diff --git a/dspy/teleprompt/avatar_optimizer.py b/dspy/teleprompt/avatar_optimizer.py
index ff6e6d4ab..500be9f24 100644
--- a/dspy/teleprompt/avatar_optimizer.py
+++ b/dspy/teleprompt/avatar_optimizer.py
@@ -1,26 +1,32 @@
+import dspy
+
+from tqdm import tqdm
from copy import deepcopy
from random import sample
-from typing import Callable, List, Tuple
-
from pydantic import BaseModel
-import dspy
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable, List, Tuple, Optional
-from dspy.evaluate import Evaluate
from dspy.teleprompt import Teleprompter
+from dspy.predict.avatar import ActionOutput
+
+
+DEFAULT_MAX_EXAMPLES = 10
class EvalResult(BaseModel):
- example: dspy.Example
+ example: dict
score: float
+ actions: Optional[List[ActionOutput]] = None
class Comparator(dspy.Signature):
- """After executing the given actions on user queries using the given instruction, some queries have yielded good, results, while others have not. I'll provide you the queries along with their, corresponding evaluation metrics:
+ """After executing the given actions on user inputs using the given instruction, some inputs have yielded good, results, while others have not. I'll provide you the inputs along with their, corresponding evaluation metrics:
Task:
-(1) Firstly, identify and contrast the patterns of queries that have achieved good, results with those that have not.
-(2) Then, review the computational logic for any inconsistencies in the previous, actions.
-(3) Lastly, specify the modification that can lead to improved performance on the, negative queries. You should focus on capturing the high-level pattern of, the queries relevant to the knowledge base schema."""
+(1) Firstly, identify and contrast the patterns of inputs that have achieved good results with those that have not.
+(2) Then, review the computational logic for any inconsistencies in the previous actions.
+(3) Lastly, specify the modification in tools used that can lead to improved performance on the negative inputs."""
instruction: str = dspy.InputField(
prefix="Instruction: ",
@@ -32,11 +38,11 @@ Task:
)
pos_input_with_metrics: List[EvalResult] = dspy.InputField(
prefix="Positive Inputs: ",
- desc="Positive inputs along with their score on a evaluation metric",
+ desc="Positive inputs along with their score on a evaluation metric and actions taken",
)
neg_input_with_metrics: List[EvalResult] = dspy.InputField(
prefix="Negative Inputs: ",
- desc="Negative inputs along with their score on a evaluation metric",
+ desc="Negative inputs along with their score on a evaluation metric and actions taken",
)
feedback: str = dspy.OutputField(
prefix="Feedback: ",
@@ -47,23 +53,18 @@ Task:
class AvatarOptimizer(Teleprompter):
def __init__(
self,
- prompt_model: dspy.LM = None,
- task_model: dspy.LM = None,
- metric: Callable = None,
+ metric: Callable,
init_temperature: float = 1.4,
max_iters: int = 10,
lower_bound: int = 0,
upper_bound: int = 1,
max_positive_inputs: int = None,
max_negative_inputs: int = None,
- metric_order: str = "max",
+ optimize_for: str = "max",
):
- self.prompt_model = prompt_model or dspy.settings.lm
- self.task_model = task_model or dspy.settings.lm
-
assert metric is not None, "`metric` argument cannot be None. Please provide a metric function."
self.metric = metric
- self.metric_order = metric_order
+ self.optimize_for = optimize_for
self.max_iters = max_iters
self.init_temperature = init_temperature
@@ -71,10 +72,58 @@ class AvatarOptimizer(Teleprompter):
self.lower_bound = lower_bound
self.upper_bound = upper_bound
- self.max_positive_inputs = max_positive_inputs
- self.max_negative_inputs = max_negative_inputs
+ self.max_positive_inputs = max_positive_inputs or DEFAULT_MAX_EXAMPLES
+ self.max_negative_inputs = max_negative_inputs or DEFAULT_MAX_EXAMPLES
self.comparator = dspy.TypedPredictor(Comparator)
+
+
+ def process_example(self, actor, example, return_outputs):
+ actor = deepcopy(actor)
+
+ try:
+ prediction = actor(**example.inputs().toDict())
+ score = metric(example, prediction)
+
+ match return_outputs:
+ case True:
+ return example, prediction, score
+ case False:
+ return score
+
+ except Exception as e:
+ print(e)
+
+ match return_outputs:
+ case True:
+ return example, None, 0
+ case False:
+ return 0
+
+
+ def multi_thread_executor(self, devset, actor, return_outputs=False, num_threads=60):
+ total_score = 0
+ total_examples = len(devset)
+ results = []
+
+ with ThreadPoolExecutor(max_workers=num_threads) as executor:
+ futures = [executor.submit(self.process_example, actor, example, return_outputs) for example in devset]
+
+ for future in tqdm(futures, total=total_examples, desc="Processing examples"):
+ result = future.result()
+ if return_outputs:
+ example, prediction, score = result
+ total_score += score
+ results.append((example, prediction, score))
+ else:
+ total_score += result
+
+ avg_metric = total_score / total_examples
+
+ if return_outputs:
+ return avg_metric, results
+ else:
+ return avg_metric
def _get_pos_neg_results(
@@ -85,26 +134,25 @@ class AvatarOptimizer(Teleprompter):
pos_inputs = []
neg_inputs = []
- _, results = self.evaluate(actor, trainset, return_outputs=True)
+ _, results = self.multi_thread_executor(trainset, actor, return_outputs=True)
- for example, _, score in results:
+ for example, prediction, score in results:
if score >= self.upper_bound:
pos_inputs.append(
EvalResult(
- example=example.inputs(),
- score=score
+ example=example.inputs().toDict(),
+ score=score,
+ actions=prediction.actions if prediction else None
)
)
elif score <= self.lower_bound:
neg_inputs.append(
EvalResult(
- example=example.inputs(),
- score=score
+ example=example.inputs().toDict(),
+ score=score,
+ actions=prediction.actions if prediction else None
)
)
- if self.verbose:
- print(f"Positive examples: {len(pos_inputs)}")
- print(f"Negative examples: {len(neg_inputs)}")
if len(pos_inputs) == 0:
raise ValueError("No positive examples found, try lowering the upper_bound or providing more training data")
@@ -112,51 +160,55 @@ class AvatarOptimizer(Teleprompter):
raise ValueError("No negative examples found, try raising the lower_bound or providing more training data")
return (pos_inputs, neg_inputs)
+
-
- def compile(self, student, *, trainset, eval_kwargs):
+ def compile(self, student, *, trainset):
actor = deepcopy(student)
best_feedback = None
- self.evaluate = Evaluate(actor, metric=self.metric, **eval_kwargs)
- best_score = self.evaluate(actor, trainset)
+ best_score = self.multi_thread_executor(trainset, actor)
print(f"Average Unoptimized score: {best_score}")
for i in range(self.max_iters):
+ print(20*'=')
print(f"Iteration {i+1}/{self.max_iters}")
- pos_results, neg_results = self._get_pos_neg_results(actor, trainset)
+ pos_inputs, neg_inputs = self._get_pos_neg_results(actor, trainset)
+ print(f"Positive examples: {len(pos_inputs)}")
+ print(f"Negative examples: {len(neg_inputs)}")
+ print(f"Sampling {self.max_positive_inputs} positive examples and {self.max_negative_inputs} negative examples")
- if self.max_positive_inputs and len(pos_results) > self.max_positive_inputs:
- pos_results = sample(pos_results, self.max_positive_inputs)
+ if self.max_positive_inputs and len(pos_inputs) > self.max_positive_inputs:
+ pos_inputs = sample(pos_inputs, self.max_positive_inputs)
- if self.max_negative_inputs and len(neg_results) > self.max_negative_inputs:
- neg_results = sample(neg_results, self.max_negative_inputs)
+ if self.max_negative_inputs and len(neg_inputs) > self.max_negative_inputs:
+ neg_inputs = sample(neg_inputs, self.max_negative_inputs)
feedback = self.comparator(
- instruction=actor.goal,
+ instruction=actor.signature.instructions,
actions=[str(tool) for tool in actor.tools],
- pos_input_with_metrics=pos_results,
- neg_input_with_metrics=neg_results
+ pos_input_with_metrics=pos_inputs,
+ neg_input_with_metrics=neg_inputs
).feedback
+ print(f"\n\nFeedback for iteration {i+1}: {feedback}\n\n")
+
if actor.feedback is None:
actor._add_feedback_field()
actor.feedback = feedback
best_feedback = feedback
else:
- if self.metric_order == "max" and self.metric(feedback) > self.metric(actor.feedback):
+ if self.optimize_for == "max" and self.metric(feedback) > self.metric(actor.feedback):
actor.feedback = feedback
best_feedback = feedback
- elif self.metric_order == "min" and self.metric(feedback) < self.metric(actor.feedback):
+ elif self.optimize_for == "min" and self.metric(feedback) < self.metric(actor.feedback):
actor.feedback = feedback
best_feedback = feedback
- avg_score = self.evaluate(actor, trainset)
+ avg_score = self.multi_thread_executor(trainset, actor)
- if self.verbose:
- print(f"Average Score: {avg_score}")
+ print(f"Average Score: {avg_score}")
print(f"Best Feedback: {best_feedback}")