Skip to content

armgit / dspy

Public Imported from GitHub
0

DSPy: The framework for programming—not prompting—language models

Python

fix coding style in dspy/evaluate (#8177)

Chen Qianc994b00ff2ea
diff --git a/dspy/evaluate/__init__.py b/dspy/evaluate/__init__.py
index a4dd669ae..24c0a2fef 100644
--- a/dspy/evaluate/__init__.py
+++ b/dspy/evaluate/__init__.py
@@ -1,8 +1,7 @@
from dspy.dsp.utils import EM, normalize_text
-
-from dspy.evaluate.metrics import answer_exact_match, answer_passage_match
+from dspy.evaluate.auto_evaluation import CompleteAndGrounded, SemanticF1
from dspy.evaluate.evaluate import Evaluate
-from dspy.evaluate.auto_evaluation import SemanticF1, CompleteAndGrounded
+from dspy.evaluate.metrics import answer_exact_match, answer_passage_match
__all__ = [
"EM",
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.py
index 5ddf66cb2..5740ba8a1 100644
--- a/dspy/evaluate/evaluate.py
+++ b/dspy/evaluate/evaluate.py
@@ -1,6 +1,6 @@
import logging
import types
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union, Tuple
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union
if TYPE_CHECKING:
import pandas as pd
@@ -25,7 +25,7 @@ except ImportError:
"""
print(obj)
- def HTML(x: str) -> str:
+ def HTML(x: str) -> str: # noqa: N802
"""
Obtain the HTML representation of the specified string.
"""
@@ -44,9 +44,10 @@ logger = logging.getLogger(__name__)
class Evaluate:
"""DSPy Evaluate class.
- This class is used to evaluate the performance of a DSPy program. Users need to provide a evaluation dataset and
+ This class is used to evaluate the performance of a DSPy program. Users need to provide a evaluation dataset and
a metric function in order to use this class. This class supports parallel evaluation on the provided dataset.
"""
+
def __init__(
self,
*,
@@ -68,8 +69,8 @@ class Evaluate:
metric (Callable): The metric function to use for evaluation.
num_threads (Optional[int]): The number of threads to use for parallel evaluation.
display_progress (bool): Whether to display progress during evaluation.
- display_table (Union[bool, int]): Whether to display the evaluation results in a table.
- If a number is passed, the evaluation results will be truncated to that number before displayed.
+ display_table (Union[bool, int]): Whether to display the evaluation results in a table.
+ If a number is passed, the evaluation results will be truncated to that number before displayed.
max_errors (int): The maximum number of errors to allow before stopping evaluation.
return_all_scores (bool): Whether to return scores for every data record in `devset`.
return_outputs (bool): Whether to return the dspy program's outputs for every data in `devset`.
@@ -119,15 +120,15 @@ class Evaluate:
Returns:
The evaluation results are returned in different formats based on the flags:
-
+
- Base return: A float percentage score (e.g., 67.30) representing overall performance
-
+
- With `return_all_scores=True`:
- Returns (overall_score, individual_scores) where individual_scores is a list of
+ Returns (overall_score, individual_scores) where individual_scores is a list of
float scores for each example in devset
-
+
- With `return_outputs=True`:
- Returns (overall_score, result_triples) where result_triples is a list of
+ Returns (overall_score, result_triples) where result_triples is a list of
(example, prediction, score) tuples for each example in devset
- With both flags=True:
@@ -181,7 +182,7 @@ class Evaluate:
metric_name = metric.__name__ if isinstance(metric, types.FunctionType) else metric.__class__.__name__
# Construct a pandas DataFrame from the results
result_df = self._construct_result_table(results, metric_name)
-
+
self._display_result_table(result_df, display_table, metric_name)
if return_all_scores and return_outputs:
@@ -192,9 +193,10 @@ class Evaluate:
return round(100 * ncorrect / ntotal, 2), results
return round(100 * ncorrect / ntotal, 2)
-
- def _construct_result_table(self, results: list[Tuple[dspy.Example, dspy.Example, Any]], metric_name: str) -> "pd.DataFrame":
+ def _construct_result_table(
+ self, results: list[Tuple[dspy.Example, dspy.Example, Any]], metric_name: str
+ ) -> "pd.DataFrame":
"""
Construct a pandas DataFrame from the specified result list.
Let's not try to change the name of this method as it may be patched by external tracing tools.
@@ -202,11 +204,12 @@ class Evaluate:
Args:
results: The list of results to construct the result DataFrame from.
metric_name: The name of the metric used for evaluation.
-
+
Returns:
The constructed pandas DataFrame.
"""
import pandas as pd
+
data = [
(
merge_dicts(example, prediction) | {"correct": score}
@@ -222,14 +225,13 @@ class Evaluate:
return result_df.rename(columns={"correct": metric_name})
-
def _display_result_table(self, result_df: "pd.DataFrame", display_table: Union[bool, int], metric_name: str):
"""
Display the specified result DataFrame in a table format.
Args:
result_df: The result DataFrame to display.
- display_table: Whether to display the evaluation results in a table.
+ display_table: Whether to display the evaluation results in a table.
If a number is passed, the evaluation results will be truncated to that number before displayed.
metric_name: The name of the metric used for evaluation.
"""
@@ -262,7 +264,7 @@ class Evaluate:
def prediction_is_dictlike(prediction):
# Downstream logic for displaying dictionary-like predictions depends solely on the predictions
# having a method called `items()` for iterating through key/value pairs
- return hasattr(prediction, "items") and callable(getattr(prediction, "items"))
+ return hasattr(prediction, "items") and callable(prediction.items)
def merge_dicts(d1, d2) -> dict:
diff --git a/dspy/evaluate/metrics.py b/dspy/evaluate/metrics.py
index c67dbf82d..dc9a3db41 100644
--- a/dspy/evaluate/metrics.py
+++ b/dspy/evaluate/metrics.py
@@ -30,13 +30,14 @@ def answer_exact_match(example, pred, trace=None, frac=1.0):
return _answer_match(pred.answer, [example.answer], frac=frac)
elif isinstance(example.answer, list):
return _answer_match(pred.answer, example.answer, frac=frac)
-
+
raise ValueError(f"Invalid answer type: {type(example.answer)}")
-def answer_passage_match(example, pred, trace=None):
+
+def answer_passage_match(example, pred, trace=None):
if isinstance(example.answer, str):
return _passage_match(pred.context, [example.answer])
elif isinstance(example.answer, list):
return _passage_match(pred.context, example.answer)
-
+
raise ValueError(f"Invalid answer type: {type(example.answer)}")