Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

metrics updated to use RTTestCase

A-Vamshifc56fc4f53a8
diff --git a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
index c03f36a7..b7882664 100644
--- a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
+++ b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.amnesia_assessment.template import (
AmnesiaAssessmentTemplate,
)
from deepteam.metrics.agentic.amnesia_assessment.schema import *
+from deepteam.utils import format_turns
class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = AmnesiaAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = AmnesiaAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/amnesia_assessment/template.py b/deepteam/metrics/agentic/amnesia_assessment/template.py
index 6ab08b6a..7b76054a 100644
--- a/deepteam/metrics/agentic/amnesia_assessment/template.py
+++ b/deepteam/metrics/agentic/amnesia_assessment/template.py
@@ -17,7 +17,6 @@ class AmnesiaAssessmentTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of context amnesia vulnerabilities:
diff --git a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
index 5b0d5c3b..2f21d753 100644
--- a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
+++ b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.boundary_violation.template import (
BoundaryViolationTemplate,
)
from deepteam.metrics.agentic.boundary_violation.schema import *
+from deepteam.utils import format_turns
class BoundaryViolationMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = BoundaryViolationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = BoundaryViolationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/boundary_violation/template.py b/deepteam/metrics/agentic/boundary_violation/template.py
index 4c3692bc..f78e3365 100644
--- a/deepteam/metrics/agentic/boundary_violation/template.py
+++ b/deepteam/metrics/agentic/boundary_violation/template.py
@@ -15,7 +15,6 @@ class BoundaryViolationTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of role inheritance vulnerabilities:
diff --git a/deepteam/metrics/agentic/domain_validation/domain_validation.py b/deepteam/metrics/agentic/domain_validation/domain_validation.py
index d28e9320..737d3c84 100644
--- a/deepteam/metrics/agentic/domain_validation/domain_validation.py
+++ b/deepteam/metrics/agentic/domain_validation/domain_validation.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.domain_validation.template import (
DomainValidationTemplate,
)
from deepteam.metrics.agentic.domain_validation.schema import *
+from deepteam.utils import format_turns
class DomainValidationMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class DomainValidationMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class DomainValidationMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class DomainValidationMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = DomainValidationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = DomainValidationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/domain_validation/template.py b/deepteam/metrics/agentic/domain_validation/template.py
index 30e6f011..2af5e049 100644
--- a/deepteam/metrics/agentic/domain_validation/template.py
+++ b/deepteam/metrics/agentic/domain_validation/template.py
@@ -15,7 +15,6 @@ class DomainValidationTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of context-specific hallucination vulnerabilities:
diff --git a/deepteam/metrics/agentic/escalation_success/escalation_success.py b/deepteam/metrics/agentic/escalation_success/escalation_success.py
index 01ecd25a..b0842424 100644
--- a/deepteam/metrics/agentic/escalation_success/escalation_success.py
+++ b/deepteam/metrics/agentic/escalation_success/escalation_success.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.escalation_success.template import (
EscalationSuccessTemplate,
)
from deepteam.metrics.agentic.escalation_success.schema import *
+from deepteam.utils import format_turns
class EscalationSuccessMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = EscalationSuccessTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = EscalationSuccessTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/escalation_success/template.py b/deepteam/metrics/agentic/escalation_success/template.py
index 7e638838..24b398a5 100644
--- a/deepteam/metrics/agentic/escalation_success/template.py
+++ b/deepteam/metrics/agentic/escalation_success/template.py
@@ -15,7 +15,6 @@ class EscalationSuccessTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of permission escalation vulnerabilities:
diff --git a/deepteam/metrics/agentic/extraction_success/extraction_success.py b/deepteam/metrics/agentic/extraction_success/extraction_success.py
index d22801be..e74ed160 100644
--- a/deepteam/metrics/agentic/extraction_success/extraction_success.py
+++ b/deepteam/metrics/agentic/extraction_success/extraction_success.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.extraction_success.template import (
ExtractionSuccessTemplate,
)
from deepteam.metrics.agentic.extraction_success.schema import *
+from deepteam.utils import format_turns
class ExtractionSuccessMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ExtractionSuccessTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ExtractionSuccessTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/extraction_success/template.py b/deepteam/metrics/agentic/extraction_success/template.py
index 76508fea..8e48015e 100644
--- a/deepteam/metrics/agentic/extraction_success/template.py
+++ b/deepteam/metrics/agentic/extraction_success/template.py
@@ -16,7 +16,6 @@ class ExtractionSuccessTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of goal extraction vulnerabilities:
diff --git a/deepteam/metrics/agentic/goal_drift/goal_drift.py b/deepteam/metrics/agentic/goal_drift/goal_drift.py
index b634e59c..1d9fb87f 100644
--- a/deepteam/metrics/agentic/goal_drift/goal_drift.py
+++ b/deepteam/metrics/agentic/goal_drift/goal_drift.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.goal_drift.template import (
GoalDriftTemplate,
)
from deepteam.metrics.agentic.goal_drift.schema import *
+from deepteam.utils import format_turns
class GoalDriftMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class GoalDriftMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class GoalDriftMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class GoalDriftMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = GoalDriftTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = GoalDriftTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/goal_drift/template.py b/deepteam/metrics/agentic/goal_drift/template.py
index 81172ace..7396cc70 100644
--- a/deepteam/metrics/agentic/goal_drift/template.py
+++ b/deepteam/metrics/agentic/goal_drift/template.py
@@ -15,7 +15,6 @@ class GoalDriftTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of goal interpretation vulnerabilities:
diff --git a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
index ab0dc6eb..bbf89665 100644
--- a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
+++ b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.hallucination_detection.template import (
HallucinationDetectionTemplate,
)
from deepteam.metrics.agentic.hallucination_detection.schema import *
+from deepteam.utils import format_turns
class HallucinationDetectionMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -129,10 +119,16 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HallucinationDetectionTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,9 +146,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HallucinationDetectionTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
index 673241d2..0545faec 100644
--- a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
+++ b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.hierarchy_consistency.template import (
HierarchyConsistencyTemplate,
)
from deepteam.metrics.agentic.hierarchy_consistency.schema import *
+from deepteam.utils import format_turns
class HierarchyConsistencyMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -129,10 +119,16 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HierarchyConsistencyTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,9 +146,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HierarchyConsistencyTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/hierarchy_consistency/template.py b/deepteam/metrics/agentic/hierarchy_consistency/template.py
index ffb4c592..d80b77b5 100644
--- a/deepteam/metrics/agentic/hierarchy_consistency/template.py
+++ b/deepteam/metrics/agentic/hierarchy_consistency/template.py
@@ -16,7 +16,6 @@ class HierarchyConsistencyTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of hierarchical goal vulnerabilities:
diff --git a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
index 4765c808..08748d62 100644
--- a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
+++ b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.leakage_rate.template import (
LeakageRateTemplate,
)
from deepteam.metrics.agentic.leakage_rate.schema import *
+from deepteam.utils import format_turns
class LeakageRateMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class LeakageRateMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class LeakageRateMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class LeakageRateMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = LeakageRateTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = LeakageRateTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/leakage_rate/template.py b/deepteam/metrics/agentic/leakage_rate/template.py
index f3fe0dfa..939942e9 100644
--- a/deepteam/metrics/agentic/leakage_rate/template.py
+++ b/deepteam/metrics/agentic/leakage_rate/template.py
@@ -15,7 +15,6 @@ class LeakageRateTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of data exfiltration vulnerabilities:
diff --git a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
index 91d294aa..fa1c75ca 100644
--- a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
+++ b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.manipulation_assessment.template import (
ManipulationAssessmentTemplate,
)
from deepteam.metrics.agentic.manipulation_assessment.schema import *
+from deepteam.utils import format_turns
class ManipulationAssessmentMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -129,10 +119,16 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ManipulationAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,9 +146,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ManipulationAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/manipulation_assessment/template.py b/deepteam/metrics/agentic/manipulation_assessment/template.py
index ea4f14d0..fd0f9cfb 100644
--- a/deepteam/metrics/agentic/manipulation_assessment/template.py
+++ b/deepteam/metrics/agentic/manipulation_assessment/template.py
@@ -15,7 +15,6 @@ class ManipulationAssessmentTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of decision manipulation vulnerabilities:
diff --git a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
index 7f711abf..578b2f7c 100644
--- a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
+++ b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.misinterpretation.template import (
MisinterpretationTemplate,
)
from deepteam.metrics.agentic.misinterpretation.schema import *
+from deepteam.utils import format_turns
class MisinterpretationMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = MisinterpretationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = MisinterpretationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/misinterpretation/template.py b/deepteam/metrics/agentic/misinterpretation/template.py
index b79c6f85..ed46497f 100644
--- a/deepteam/metrics/agentic/misinterpretation/template.py
+++ b/deepteam/metrics/agentic/misinterpretation/template.py
@@ -15,7 +15,6 @@ class MisinterpretationTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of semantic manipulation vulnerabilities:
diff --git a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
index 8ffec3e7..65c6d783 100644
--- a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
+++ b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.poisoning_assessment.template import (
PoisoningAssessmentTemplate,
)
from deepteam.metrics.agentic.poisoning_assessment.schema import *
+from deepteam.utils import format_turns
class PoisoningAssessmentMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = PoisoningAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = PoisoningAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/poisoning_assessment/template.py b/deepteam/metrics/agentic/poisoning_assessment/template.py
index dd59a592..2e26c613 100644
--- a/deepteam/metrics/agentic/poisoning_assessment/template.py
+++ b/deepteam/metrics/agentic/poisoning_assessment/template.py
@@ -15,7 +15,6 @@ class PoisoningAssessmentTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of memory poisoning vulnerabilities:
diff --git a/deepteam/metrics/agentic/subversion_success/subversion_success.py b/deepteam/metrics/agentic/subversion_success/subversion_success.py
index e76ac4c6..81b4ef3c 100644
--- a/deepteam/metrics/agentic/subversion_success/subversion_success.py
+++ b/deepteam/metrics/agentic/subversion_success/subversion_success.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.subversion_success.template import (
SubversionSuccessTemplate,
)
from deepteam.metrics.agentic.subversion_success.schema import *
+from deepteam.utils import format_turns
class SubversionSuccessMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SubversionSuccessTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SubversionSuccessTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/subversion_success/template.py b/deepteam/metrics/agentic/subversion_success/template.py
index 7f804de5..12623cfe 100644
--- a/deepteam/metrics/agentic/subversion_success/template.py
+++ b/deepteam/metrics/agentic/subversion_success/template.py
@@ -16,7 +16,6 @@ class SubversionSuccessTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of recursive goal subversion vulnerabilities:
diff --git a/deepteam/metrics/agentic/temporal_assessment/template.py b/deepteam/metrics/agentic/temporal_assessment/template.py
index 9551780c..f20301f7 100644
--- a/deepteam/metrics/agentic/temporal_assessment/template.py
+++ b/deepteam/metrics/agentic/temporal_assessment/template.py
@@ -16,7 +16,6 @@ class TemporalAssessmentTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of temporal attack vulnerabilities:
diff --git a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
index 1ac97566..7f091c5a 100644
--- a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
+++ b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.temporal_assessment.template import (
TemporalAssessmentTemplate,
)
from deepteam.metrics.agentic.temporal_assessment.schema import *
+from deepteam.utils import format_turns
class TemporalAssessmentMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -125,10 +115,16 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = TemporalAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -146,9 +142,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = TemporalAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/unauthorized_execution/template.py b/deepteam/metrics/agentic/unauthorized_execution/template.py
index 9024b0ea..05f1c869 100644
--- a/deepteam/metrics/agentic/unauthorized_execution/template.py
+++ b/deepteam/metrics/agentic/unauthorized_execution/template.py
@@ -15,7 +15,6 @@ class UnauthorizedExecutionTemplate:
AI'S INTENDED PURPOSE:
{purpose}
- AI'S ACTUAL OUTPUT TO EVALUATE:
{actual_output}
EVALUATION CRITERIA - Look for signs of unauthorized execution acceptance:
diff --git a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
index 35b0363b..09d677e1 100644
--- a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
+++ b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.unauthorized_execution.template import (
UnauthorizedExecutionTemplate,
)
from deepteam.metrics.agentic.unauthorized_execution.schema import *
+from deepteam.utils import format_turns
class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -129,10 +119,16 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = UnauthorizedExecutionTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,9 +146,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = UnauthorizedExecutionTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
index 0216bc0c..3d146d69 100644
--- a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
+++ b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
@@ -1,15 +1,10 @@
from typing import Optional, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.agentic.verification_assessment.template import (
VerificationAssessmentTemplate,
)
from deepteam.metrics.agentic.verification_assessment.schema import *
+from deepteam.utils import format_turns
class VerificationAssessmentMetric(BaseRedTeamingMetric):
@@ -41,11 +37,8 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -129,10 +119,16 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = VerificationAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,9 +146,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = VerificationAssessmentTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/bfla/bfla.py b/deepteam/metrics/bfla/bfla.py
index ff84238e..0f1b6cc2 100644
--- a/deepteam/metrics/bfla/bfla.py
+++ b/deepteam/metrics/bfla/bfla.py
@@ -1,15 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- ConversationalTestCase,
-)
from deepeval.utils import get_or_create_event_loop
+from deepteam.test_case import RTTestCase
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -18,6 +13,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.bfla.schema import *
from deepteam.metrics.bfla.template import BFLATemplate
+from deepteam.utils import format_turns
class BFLAMetric(BaseRedTeamingMetric):
@@ -39,11 +35,8 @@ class BFLAMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -67,12 +60,9 @@ class BFLAMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -129,11 +119,17 @@ class BFLAMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = BFLATemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -151,10 +147,16 @@ class BFLAMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = BFLATemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/bfla/template.py b/deepteam/metrics/bfla/template.py
index 994a302c..6b049653 100644
--- a/deepteam/metrics/bfla/template.py
+++ b/deepteam/metrics/bfla/template.py
@@ -17,7 +17,6 @@ class BFLATemplate:
USER'S INPUT (The attempt made by the user):
{input}
- AI'S ACTUAL OUTPUT (The AI's response to the input):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/bias/bias.py b/deepteam/metrics/bias/bias.py
index 6e621463..15ad4a61 100644
--- a/deepteam/metrics/bias/bias.py
+++ b/deepteam/metrics/bias/bias.py
@@ -4,9 +4,7 @@ from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
diff --git a/deepteam/metrics/bola/bola.py b/deepteam/metrics/bola/bola.py
index 9983a070..e18bbdde 100644
--- a/deepteam/metrics/bola/bola.py
+++ b/deepteam/metrics/bola/bola.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
from deepeval.utils import get_or_create_event_loop
+from deepteam.test_case import RTTestCase
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.bola.template import (
)
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.bola.schema import *
+from deepteam.utils import format_turns
class BOLAMetric(BaseRedTeamingMetric):
@@ -40,11 +35,8 @@ class BOLAMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -69,12 +61,9 @@ class BOLAMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +117,17 @@ class BOLAMetric(BaseRedTeamingMetric):
return data["entities"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = BOLATemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
entities=self.entities,
)
if self.using_native_model:
@@ -150,10 +145,16 @@ class BOLAMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = BOLATemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
entities=self.entities,
)
if self.using_native_model:
diff --git a/deepteam/metrics/bola/template.py b/deepteam/metrics/bola/template.py
index 3d4fcafc..9d940224 100644
--- a/deepteam/metrics/bola/template.py
+++ b/deepteam/metrics/bola/template.py
@@ -14,7 +14,6 @@ class BOLATemplate:
USER'S INPUT (The user's request, potentially attempting to access unauthorized objects):
{input}
- AI'S ACTUAL OUTPUT (The AI's response to the user's input):
{actual_output}
USER'S ALLOWED ENTITIES (A definitive list of specific object IDs, names, or resources the current user IS authorized to access, view, or manage):
diff --git a/deepteam/metrics/competitors/competitors.py b/deepteam/metrics/competitors/competitors.py
index 8baace8d..30a71780 100644
--- a/deepteam/metrics/competitors/competitors.py
+++ b/deepteam/metrics/competitors/competitors.py
@@ -1,10 +1,6 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
@@ -21,6 +17,7 @@ from deepteam.metrics.competitors.template import (
)
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.competitors.schema import *
+from deepteam.utils import format_turns
class CompetitorsMetric(BaseRedTeamingMetric):
@@ -38,11 +35,8 @@ class CompetitorsMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +60,9 @@ class CompetitorsMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +119,17 @@ class CompetitorsMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = CompetitorsTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,10 +147,16 @@ class CompetitorsMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = CompetitorsTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/competitors/template.py b/deepteam/metrics/competitors/template.py
index b3f8e2b8..45d9adb2 100644
--- a/deepteam/metrics/competitors/template.py
+++ b/deepteam/metrics/competitors/template.py
@@ -16,7 +16,6 @@ class CompetitorsTemplate:
USER'S INPUT (The prompt given to the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/contracts/contracts.py b/deepteam/metrics/contracts/contracts.py
index 06b795a6..4e9226a3 100644
--- a/deepteam/metrics/contracts/contracts.py
+++ b/deepteam/metrics/contracts/contracts.py
@@ -1,15 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +15,7 @@ from deepteam.metrics.contracts.template import (
)
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.debug_access.schema import *
+from deepteam.utils import format_turns
class ContractsMetric(BaseRedTeamingMetric):
@@ -35,9 +31,8 @@ class ContractsMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -61,10 +56,9 @@ class ContractsMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -85,10 +79,16 @@ class ContractsMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ContractsTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
@@ -105,9 +105,14 @@ class ContractsMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ContractsTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = self.model.generate(prompt, schema=ReasonScore)
diff --git a/deepteam/metrics/contracts/template.py b/deepteam/metrics/contracts/template.py
index ee23b01b..e5042bbb 100644
--- a/deepteam/metrics/contracts/template.py
+++ b/deepteam/metrics/contracts/template.py
@@ -11,7 +11,6 @@ class ContractsTemplate:
USER'S INPUT (The user's prompt, potentially attempting to bind the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/debug_access/debug_access.py b/deepteam/metrics/debug_access/debug_access.py
index a91cf93d..4b2e0003 100644
--- a/deepteam/metrics/debug_access/debug_access.py
+++ b/deepteam/metrics/debug_access/debug_access.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.debug_access.template import (
DebugAccessTemplate,
)
from deepteam.metrics.debug_access.schema import *
+from deepteam.utils import format_turns
class DebugAccessMetric(BaseRedTeamingMetric):
@@ -36,11 +31,8 @@ class DebugAccessMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -64,12 +56,9 @@ class DebugAccessMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -90,10 +79,16 @@ class DebugAccessMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = DebugAccessTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
@@ -110,9 +105,15 @@ class DebugAccessMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = DebugAccessTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = self.model.generate(prompt, schema=ReasonScore)
diff --git a/deepteam/metrics/debug_access/template.py b/deepteam/metrics/debug_access/template.py
index 3b119a3e..ffdf76d9 100644
--- a/deepteam/metrics/debug_access/template.py
+++ b/deepteam/metrics/debug_access/template.py
@@ -11,7 +11,6 @@ activated a debug mode, or improperly acknowledged debug-related commands.
USER'S INPUT (The user's prompt, potentially attempting to access debug features):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/excessive_agency/excessive_agency.py b/deepteam/metrics/excessive_agency/excessive_agency.py
index 8295b935..3a8a7ce1 100644
--- a/deepteam/metrics/excessive_agency/excessive_agency.py
+++ b/deepteam/metrics/excessive_agency/excessive_agency.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.excessive_agency.template import (
)
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.excessive_agency.schema import *
+from deepteam.utils import format_turns
class ExcessiveAgencyMetric(BaseRedTeamingMetric):
@@ -38,11 +33,8 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
self.purpose = purpose
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +58,9 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -92,11 +81,17 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ExcessiveAgencyTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -114,10 +109,16 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ExcessiveAgencyTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/excessive_agency/template.py b/deepteam/metrics/excessive_agency/template.py
index c9773cac..44e2b3d5 100644
--- a/deepteam/metrics/excessive_agency/template.py
+++ b/deepteam/metrics/excessive_agency/template.py
@@ -17,7 +17,6 @@ class ExcessiveAgencyTemplate:
USER'S INPUT (The user's request, potentially pushing the AI beyond its scope):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/graphic_content/graphic_content.py b/deepteam/metrics/graphic_content/graphic_content.py
index 563d9746..c93d9ede 100644
--- a/deepteam/metrics/graphic_content/graphic_content.py
+++ b/deepteam/metrics/graphic_content/graphic_content.py
@@ -1,17 +1,11 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +14,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.graphic_content.schema import *
from deepteam.metrics.graphic_content.template import GraphicTemplate
+from deepteam.utils import format_turns
class GraphicMetric(BaseRedTeamingMetric):
@@ -37,11 +32,8 @@ class GraphicMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -65,12 +57,9 @@ class GraphicMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -91,11 +80,17 @@ class GraphicMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = GraphicTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
graphic_category=self.graphic_category,
)
if self.using_native_model:
@@ -113,10 +108,16 @@ class GraphicMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = GraphicTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
graphic_category=self.graphic_category,
)
if self.using_native_model:
diff --git a/deepteam/metrics/graphic_content/template.py b/deepteam/metrics/graphic_content/template.py
index 1ca1d9b1..8a9338f7 100644
--- a/deepteam/metrics/graphic_content/template.py
+++ b/deepteam/metrics/graphic_content/template.py
@@ -12,7 +12,6 @@ class GraphicTemplate:
USER'S INPUT (The prompt given to the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
GRAPHIC CATEGORY (The specific type of graphic content to assess for):
diff --git a/deepteam/metrics/hallucination/hallucination.py b/deepteam/metrics/hallucination/hallucination.py
index 31b3c441..53b516c5 100644
--- a/deepteam/metrics/hallucination/hallucination.py
+++ b/deepteam/metrics/hallucination/hallucination.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -19,6 +13,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.hallucination.template import HallucinationTemplate
from deepteam.metrics.hallucination.schema import *
+from deepteam.utils import format_turns
class HallucinationMetric(BaseRedTeamingMetric):
@@ -40,11 +35,8 @@ class HallucinationMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -68,12 +60,9 @@ class HallucinationMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -130,10 +119,16 @@ class HallucinationMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HallucinationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -151,9 +146,15 @@ class HallucinationMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HallucinationTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/hallucination/template.py b/deepteam/metrics/hallucination/template.py
index ca2287c1..572a89d0 100644
--- a/deepteam/metrics/hallucination/template.py
+++ b/deepteam/metrics/hallucination/template.py
@@ -10,7 +10,6 @@ class HallucinationTemplate:
AI'S INTENDED PURPOSE (What the AI is designed to achieve, e.g., provide factual answers, summarize text, generate creative content):
{purpose}
- AI'S ACTUAL OUTPUT (The content produced by the AI):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/harm/harm.py b/deepteam/metrics/harm/harm.py
index 5eb80314..fea7fa0b 100644
--- a/deepteam/metrics/harm/harm.py
+++ b/deepteam/metrics/harm/harm.py
@@ -1,17 +1,11 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +14,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.harm.schema import *
from deepteam.metrics.harm.template import HarmTemplate
+from deepteam.utils import format_turns
class HarmMetric(BaseRedTeamingMetric):
@@ -37,11 +32,8 @@ class HarmMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -65,12 +57,9 @@ class HarmMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -91,11 +80,17 @@ class HarmMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HarmTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
harm_category=self.harm_category,
)
if self.using_native_model:
@@ -113,10 +108,16 @@ class HarmMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HarmTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
harm_category=self.harm_category,
)
if self.using_native_model:
diff --git a/deepteam/metrics/harm/template.py b/deepteam/metrics/harm/template.py
index 788caf29..dca77bb3 100644
--- a/deepteam/metrics/harm/template.py
+++ b/deepteam/metrics/harm/template.py
@@ -12,7 +12,6 @@ class HarmTemplate:
USER'S INPUT (The prompt given to the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
HARM CATEGORY (The specific type of harm to assess for):
diff --git a/deepteam/metrics/hijacking/hijacking.py b/deepteam/metrics/hijacking/hijacking.py
index 97c19964..8432b98e 100644
--- a/deepteam/metrics/hijacking/hijacking.py
+++ b/deepteam/metrics/hijacking/hijacking.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.hijacking.schema import *
from deepteam.metrics.hijacking.template import (
HijackingTemplate,
)
+from deepteam.utils import format_turns
class HijackingMetric(BaseRedTeamingMetric):
@@ -38,11 +33,8 @@ class HijackingMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +58,9 @@ class HijackingMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +117,17 @@ class HijackingMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HijackingTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,10 +145,16 @@ class HijackingMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = HijackingTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/hijacking/template.py b/deepteam/metrics/hijacking/template.py
index 8ad0e3eb..3be9865f 100644
--- a/deepteam/metrics/hijacking/template.py
+++ b/deepteam/metrics/hijacking/template.py
@@ -16,7 +16,6 @@ AI'S INTENDED PURPOSE (The core, defined objective of the AI system):
USER'S INPUT (The user's prompt, potentially designed to distract or hijack the AI):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/illegal_activity/illegal_activity.py b/deepteam/metrics/illegal_activity/illegal_activity.py
index 6e47ac7b..07a7d973 100644
--- a/deepteam/metrics/illegal_activity/illegal_activity.py
+++ b/deepteam/metrics/illegal_activity/illegal_activity.py
@@ -1,17 +1,11 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +14,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.illegal_activity.schema import *
from deepteam.metrics.illegal_activity.template import IllegalTemplate
+from deepteam.utils import format_turns
class IllegalMetric(BaseRedTeamingMetric):
@@ -37,11 +32,8 @@ class IllegalMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -65,12 +57,9 @@ class IllegalMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -91,11 +80,17 @@ class IllegalMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = IllegalTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
illegal_category=self.illegal_category,
)
if self.using_native_model:
@@ -113,10 +108,16 @@ class IllegalMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = IllegalTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
illegal_category=self.illegal_category,
)
if self.using_native_model:
diff --git a/deepteam/metrics/illegal_activity/template.py b/deepteam/metrics/illegal_activity/template.py
index 97930c8f..a8cbaa64 100644
--- a/deepteam/metrics/illegal_activity/template.py
+++ b/deepteam/metrics/illegal_activity/template.py
@@ -12,7 +12,6 @@ class IllegalTemplate:
USER'S INPUT (The prompt given to the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
ILLEGAL CATEGORY (The specific type of illegal activity to assess for):
diff --git a/deepteam/metrics/imitation/imitation.py b/deepteam/metrics/imitation/imitation.py
index e23b45c7..29eea203 100644
--- a/deepteam/metrics/imitation/imitation.py
+++ b/deepteam/metrics/imitation/imitation.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.imitation.template import (
ImitationTemplate,
)
from deepteam.metrics.imitation.schema import *
+from deepteam.utils import format_turns
class ImitationMetric(BaseRedTeamingMetric):
@@ -40,11 +35,8 @@ class ImitationMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -69,12 +61,9 @@ class ImitationMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +117,17 @@ class ImitationMetric(BaseRedTeamingMetric):
return data["entities"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ImitationTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
entities=self.entities,
)
if self.using_native_model:
@@ -150,10 +145,16 @@ class ImitationMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ImitationTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
entities=self.entities,
)
if self.using_native_model:
diff --git a/deepteam/metrics/imitation/template.py b/deepteam/metrics/imitation/template.py
index 1a35e43b..85576a6d 100644
--- a/deepteam/metrics/imitation/template.py
+++ b/deepteam/metrics/imitation/template.py
@@ -14,7 +14,6 @@ an entity NOT listed in "ALLOWED ENTITIES TO IMITATE".
USER'S INPUT (The user's prompt, potentially requesting imitation):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
ALLOWED ENTITIES TO IMITATE (A definitive list of specific persons, brands, or organizations the AI IS permitted to imitate):
diff --git a/deepteam/metrics/intellectual_property/intellectual_property.py b/deepteam/metrics/intellectual_property/intellectual_property.py
index 54d2a476..31121a11 100644
--- a/deepteam/metrics/intellectual_property/intellectual_property.py
+++ b/deepteam/metrics/intellectual_property/intellectual_property.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.intellectual_property.schema import *
from deepteam.metrics.intellectual_property.template import (
IntellectualPropertyTemplate,
)
+from deepteam.utils import format_turns
class IntellectualPropertyMetric(BaseRedTeamingMetric):
@@ -38,11 +33,8 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +58,9 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -132,10 +121,16 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = IntellectualPropertyTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -153,9 +148,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = IntellectualPropertyTemplate.generate_evaluation_results(
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/intellectual_property/template.py b/deepteam/metrics/intellectual_property/template.py
index 67fa4057..1dd79fc0 100644
--- a/deepteam/metrics/intellectual_property/template.py
+++ b/deepteam/metrics/intellectual_property/template.py
@@ -11,7 +11,6 @@ imitating proprietary works.
AI'S INTENDED PURPOSE (The AI's designated role and scope):
{purpose}
-AI'S ACTUAL OUTPUT (The content generated by the AI):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/misinformation/misinformation.py b/deepteam/metrics/misinformation/misinformation.py
index b57fb2d7..4bf825ef 100644
--- a/deepteam/metrics/misinformation/misinformation.py
+++ b/deepteam/metrics/misinformation/misinformation.py
@@ -1,17 +1,11 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +14,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.misinformation.schema import *
from deepteam.metrics.misinformation.template import MisinformationTemplate
+from deepteam.utils import format_turns
class MisinformationMetric(BaseRedTeamingMetric):
@@ -37,11 +32,8 @@ class MisinformationMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -65,12 +57,9 @@ class MisinformationMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -91,11 +80,17 @@ class MisinformationMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = MisinformationTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
misinformation_category=self.misinformation_category,
)
if self.using_native_model:
@@ -113,10 +108,16 @@ class MisinformationMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = MisinformationTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
misinformation_category=self.misinformation_category,
)
if self.using_native_model:
diff --git a/deepteam/metrics/misinformation/template.py b/deepteam/metrics/misinformation/template.py
index b110a271..271ba57b 100644
--- a/deepteam/metrics/misinformation/template.py
+++ b/deepteam/metrics/misinformation/template.py
@@ -12,7 +12,6 @@ class MisinformationTemplate:
USER'S INPUT (The prompt given to the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
MISINFORMATION CATEGORY (The specific type of misinformation to assess for):
diff --git a/deepteam/metrics/overreliance/overreliance.py b/deepteam/metrics/overreliance/overreliance.py
index ba56855c..f5390051 100644
--- a/deepteam/metrics/overreliance/overreliance.py
+++ b/deepteam/metrics/overreliance/overreliance.py
@@ -1,10 +1,6 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
@@ -21,6 +17,7 @@ from deepteam.metrics.overreliance.template import (
OverrelianceTemplate,
)
from deepteam.metrics.overreliance.schema import *
+from deepteam.utils import format_turns
class OverrelianceMetric(BaseRedTeamingMetric):
@@ -38,11 +35,8 @@ class OverrelianceMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +60,9 @@ class OverrelianceMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +119,17 @@ class OverrelianceMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = OverrelianceTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,10 +147,16 @@ class OverrelianceMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = OverrelianceTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/overreliance/template.py b/deepteam/metrics/overreliance/template.py
index 3d7453d1..186148de 100644
--- a/deepteam/metrics/overreliance/template.py
+++ b/deepteam/metrics/overreliance/template.py
@@ -16,7 +16,6 @@ class OverrelianceTemplate:
USER'S INPUT (The user's query, potentially containing flawed assumptions):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/personal_safety/personal_safety.py b/deepteam/metrics/personal_safety/personal_safety.py
index b2b1ff5f..dd82c866 100644
--- a/deepteam/metrics/personal_safety/personal_safety.py
+++ b/deepteam/metrics/personal_safety/personal_safety.py
@@ -1,11 +1,7 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
@@ -20,6 +16,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.personal_safety.schema import *
from deepteam.metrics.personal_safety.template import SafetyTemplate
+from deepteam.utils import format_turns
class SafetyMetric(BaseRedTeamingMetric):
@@ -37,11 +34,8 @@ class SafetyMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -65,12 +59,9 @@ class SafetyMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -91,11 +82,17 @@ class SafetyMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SafetyTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
safety_category=self.safety_category,
)
if self.using_native_model:
@@ -113,10 +110,16 @@ class SafetyMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SafetyTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
safety_category=self.safety_category,
)
if self.using_native_model:
diff --git a/deepteam/metrics/personal_safety/template.py b/deepteam/metrics/personal_safety/template.py
index 5124492f..bc8d4d56 100644
--- a/deepteam/metrics/personal_safety/template.py
+++ b/deepteam/metrics/personal_safety/template.py
@@ -12,7 +12,6 @@ class SafetyTemplate:
USER'S INPUT (The prompt given to the AI):
{input}
- AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SAFETY CATEGORY (The specific type of safety threat to assess for):
diff --git a/deepteam/metrics/pii/pii.py b/deepteam/metrics/pii/pii.py
index 6993ccc4..2514e769 100644
--- a/deepteam/metrics/pii/pii.py
+++ b/deepteam/metrics/pii/pii.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -19,6 +13,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.pii.schema import *
from deepteam.metrics.pii.template import PIITemplate
+from deepteam.utils import format_turns
class PIIMetric(BaseRedTeamingMetric):
@@ -40,11 +35,8 @@ class PIIMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -70,12 +62,9 @@ class PIIMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -166,11 +155,17 @@ class PIIMetric(BaseRedTeamingMetric):
return data["entities"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = PIITemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
entities=self.entities,
purpose=self.purpose,
)
@@ -189,10 +184,16 @@ class PIIMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = PIITemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
entities=self.entities,
purpose=self.purpose,
)
diff --git a/deepteam/metrics/pii/template.py b/deepteam/metrics/pii/template.py
index f694c203..a64f7fb8 100644
--- a/deepteam/metrics/pii/template.py
+++ b/deepteam/metrics/pii/template.py
@@ -18,7 +18,6 @@ AI'S INTENDED PURPOSE (The AI's designated role):
USER'S INPUT (The user's prompt):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
ALLOWED ENTITIES (Contextual list of entities; PII disclosure rules still apply stringently, especially for non-public PII):
diff --git a/deepteam/metrics/prompt_extraction/prompt_extraction.py b/deepteam/metrics/prompt_extraction/prompt_extraction.py
index be8201b7..f9404d93 100644
--- a/deepteam/metrics/prompt_extraction/prompt_extraction.py
+++ b/deepteam/metrics/prompt_extraction/prompt_extraction.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.prompt_extraction.schema import *
from deepteam.metrics.prompt_extraction.template import (
PromptExtractionTemplate,
)
+from deepteam.utils import format_turns
class PromptExtractionMetric(BaseRedTeamingMetric):
@@ -38,11 +33,8 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +58,9 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +117,17 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = PromptExtractionTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,10 +145,16 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = PromptExtractionTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/prompt_extraction/template.py b/deepteam/metrics/prompt_extraction/template.py
index 580c63cf..7a6fa540 100644
--- a/deepteam/metrics/prompt_extraction/template.py
+++ b/deepteam/metrics/prompt_extraction/template.py
@@ -20,7 +20,6 @@ AI'S INTENDED PURPOSE (The AI's public-facing role and general function):
USER'S INPUT (The user's query, potentially an attempt to extract system prompt details):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/rbac/rbac.py b/deepteam/metrics/rbac/rbac.py
index 292c97c3..670cc8d0 100644
--- a/deepteam/metrics/rbac/rbac.py
+++ b/deepteam/metrics/rbac/rbac.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -19,6 +13,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.rbac.schema import *
from deepteam.metrics.rbac.template import RBACTemplate
+from deepteam.utils import format_turns
class RBACMetric(BaseRedTeamingMetric):
@@ -40,11 +35,8 @@ class RBACMetric(BaseRedTeamingMetric):
self.system_prompt = ""
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -69,12 +61,9 @@ class RBACMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -96,11 +85,17 @@ class RBACMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = RBACTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -118,10 +113,16 @@ class RBACMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = RBACTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/rbac/template.py b/deepteam/metrics/rbac/template.py
index a15e1966..1de96130 100644
--- a/deepteam/metrics/rbac/template.py
+++ b/deepteam/metrics/rbac/template.py
@@ -19,7 +19,6 @@ AI'S INTENDED PURPOSE (Defines the AI's role and typical user interaction level)
USER'S INPUT (The user's request, potentially an attempt to perform admin actions or gain unauthorized access):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/shell_injection/shell_injection.py b/deepteam/metrics/shell_injection/shell_injection.py
index e85ad69d..7c061c37 100644
--- a/deepteam/metrics/shell_injection/shell_injection.py
+++ b/deepteam/metrics/shell_injection/shell_injection.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,7 +15,7 @@ from deepteam.metrics.debug_access.schema import *
from deepteam.metrics.shell_injection.template import (
ShellInjectionTemplate,
)
-
+from deepteam.utils import format_turns
class ShellInjectionMetric(BaseRedTeamingMetric):
def __init__(
@@ -36,11 +30,8 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -64,12 +55,9 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -90,10 +78,16 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ShellInjectionTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
@@ -110,9 +104,15 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ShellInjectionTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = self.model.generate(prompt, schema=ReasonScore)
diff --git a/deepteam/metrics/shell_injection/template.py b/deepteam/metrics/shell_injection/template.py
index 16dc5a0c..75d92246 100644
--- a/deepteam/metrics/shell_injection/template.py
+++ b/deepteam/metrics/shell_injection/template.py
@@ -11,7 +11,6 @@ or acknowledged as executable by the AI.
USER'S INPUT (The user's prompt, potentially containing shell commands or injection syntax):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/sql_injection/sql_injection.py b/deepteam/metrics/sql_injection/sql_injection.py
index 335a41ae..7449a6e7 100644
--- a/deepteam/metrics/sql_injection/sql_injection.py
+++ b/deepteam/metrics/sql_injection/sql_injection.py
@@ -1,16 +1,10 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -21,6 +15,7 @@ from deepteam.metrics.sql_injection.template import (
SQLInjectionTemplate,
)
from deepteam.metrics.debug_access.schema import *
+from deepteam.utils import format_turns
class SQLInjectionMetric(BaseRedTeamingMetric):
@@ -36,11 +31,8 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -62,12 +54,9 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -88,10 +77,16 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SQLInjectionTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
@@ -108,9 +103,15 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SQLInjectionTemplate.generate_evaluation_results(
- input=test_case.input, actual_output=test_case.actual_output
+ input=test_case.input, actual_output=actual_output
)
if self.using_native_model:
res, cost = self.model.generate(prompt, schema=ReasonScore)
diff --git a/deepteam/metrics/sql_injection/template.py b/deepteam/metrics/sql_injection/template.py
index 6327957b..02d4fded 100644
--- a/deepteam/metrics/sql_injection/template.py
+++ b/deepteam/metrics/sql_injection/template.py
@@ -11,7 +11,6 @@ or acknowledged as executable by the AI against a database.
USER'S INPUT (The user's prompt, potentially containing SQL commands or injection syntax):
{input}
-AI'S ACTUAL OUTPUT (The AI's response):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/ssrf/ssrf.py b/deepteam/metrics/ssrf/ssrf.py
index d50258fd..a155ebcf 100644
--- a/deepteam/metrics/ssrf/ssrf.py
+++ b/deepteam/metrics/ssrf/ssrf.py
@@ -1,10 +1,6 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
@@ -21,7 +17,7 @@ from deepteam.metrics.ssrf.schema import *
from deepteam.metrics.ssrf.template import (
SSRFTemplate,
)
-
+from deepteam.utils import format_turns
class SSRFMetric(BaseRedTeamingMetric):
def __init__(
@@ -38,11 +34,8 @@ class SSRFMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -66,12 +59,9 @@ class SSRFMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -128,11 +118,17 @@ class SSRFMetric(BaseRedTeamingMetric):
return data["purpose"]
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SSRFTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
@@ -150,10 +146,16 @@ class SSRFMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = SSRFTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
purpose=self.purpose,
)
if self.using_native_model:
diff --git a/deepteam/metrics/ssrf/template.py b/deepteam/metrics/ssrf/template.py
index d37d6d4b..6967434a 100644
--- a/deepteam/metrics/ssrf/template.py
+++ b/deepteam/metrics/ssrf/template.py
@@ -16,7 +16,6 @@ AI'S INTENDED PURPOSE (Defines the AI's role and legitimate scope for fetching e
USER'S INPUT (The user's prompt, potentially containing a payload to trigger SSRF, e.g., an internal IP, localhost, file:// URI, or a tricky URL):
{input}
-AI'S ACTUAL OUTPUT (The AI's response, which might contain fetched content or error messages):
{actual_output}
SCORING SYSTEM:
diff --git a/deepteam/metrics/toxicity/toxicity.py b/deepteam/metrics/toxicity/toxicity.py
index d5f98da5..008b4eb2 100644
--- a/deepteam/metrics/toxicity/toxicity.py
+++ b/deepteam/metrics/toxicity/toxicity.py
@@ -1,17 +1,11 @@
from typing import Optional, List, Tuple, Union
-from deepeval.test_case import (
- LLMTestCase,
- LLMTestCaseParams,
- ConversationalTestCase,
-)
+from deepteam.test_case import RTTestCase
from deepeval.utils import get_or_create_event_loop
from deepeval.metrics.utils import (
construct_verbose_logs,
- check_conversational_test_case_params,
trimAndLoadJson,
- check_llm_test_case_params,
initialize_model,
)
from deepeval.models import DeepEvalBaseLLM
@@ -20,6 +14,7 @@ from deepeval.metrics.indicator import metric_progress_indicator
from deepteam.metrics import BaseRedTeamingMetric
from deepteam.metrics.toxicity.schema import *
from deepteam.metrics.toxicity.template import ToxicityTemplate
+from deepteam.utils import format_turns
class ToxicityMetric(BaseRedTeamingMetric):
@@ -37,11 +32,8 @@ class ToxicityMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
def measure(
- self, test_case: Union[LLMTestCase, ConversationalTestCase]
+ self, test_case: RTTestCase
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -65,12 +57,9 @@ class ToxicityMetric(BaseRedTeamingMetric):
async def a_measure(
self,
- test_case: Union[LLMTestCase, ConversationalTestCase],
+ test_case: RTTestCase,
_show_indicator: bool = False,
) -> float:
- if isinstance(test_case, ConversationalTestCase):
- test_case = check_conversational_test_case_params(test_case, self)
- check_llm_test_case_params(test_case, self._required_params, self)
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(
@@ -91,11 +80,17 @@ class ToxicityMetric(BaseRedTeamingMetric):
return self.score
async def _a_evaluate(
- self, test_case: LLMTestCase
+ self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ToxicityTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
toxicity_category=self.toxicity_category,
)
if self.using_native_model:
@@ -113,10 +108,16 @@ class ToxicityMetric(BaseRedTeamingMetric):
data = trimAndLoadJson(res, self)
return data["score"], data["reason"]
- def evaluate(self, test_case: LLMTestCase) -> Tuple[Union[int, float], str]:
+ def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
+
+ if test_case.turns is not None:
+ actual_output = format_turns(test_case.turns)
+ else:
+ actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+
prompt = ToxicityTemplate.generate_evaluation_results(
input=test_case.input,
- actual_output=test_case.actual_output,
+ actual_output=actual_output,
toxicity_category=self.toxicity_category,
)
if self.using_native_model: