DeepTeam is a framework to red team LLMs and AI agents.
holy reformat
penguine06b51994e408
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex 50e6cb9f..2c4b0c38 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -170,9 +170,7 @@ class AttackSimulator:enhanced_test_cases.extend(await asyncio.gather(*[- asyncio.create_task(- throttled_attack_method(test_case)- )+ asyncio.create_task(throttled_attack_method(test_case))for test_case in test_cases])@@ -346,9 +344,7 @@ class AttackSimulator:raiseexcept Exception as e:if ignore_errors:- test_case.error = (- f"Error enhancing regular attack: {str(e)}"- )+ test_case.error = f"Error enhancing regular attack: {str(e)}"return test_caseelse:raisediff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.pyindex f26ed4d1..4122afa0 100644--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py@@ -43,8 +43,12 @@ class BadLikertJudge(BaseAttack):self.attacks = attacksif self.attacks is not None:- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")+ if not isinstance(self.attacks, list) or not all(+ attack.multi_turn == False for attack in self.attacks+ ):+ raise ValueError(+ "The 'attacks' passed must be a list of single-turn attacks"+ )def _get_turns(self,@@ -140,7 +144,9 @@ class BadLikertJudge(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- enhanced_attack = enhance_attack(attack, enhanced_attack, self.simulator_model)+ enhanced_attack = enhance_attack(+ attack, enhanced_attack, self.simulator_model+ )assistant_response = model_callback(current_attack, turns)turns.append(RTTurn(role="user", content=current_attack))@@ -246,7 +252,9 @@ class BadLikertJudge(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- enhanced_attack = await a_enhance_attack(attack, enhanced_attack, self.simulator_model)+ enhanced_attack = await a_enhance_attack(+ attack, enhanced_attack, self.simulator_model+ )assistant_response = await model_callback(current_attack, turns)turns.append(RTTurn(role="user", content=current_attack))diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.pyindex e01433c0..e6d1ed0e 100644--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py@@ -71,8 +71,12 @@ class CrescendoJailbreaking(BaseAttack):self.attacks = attacksif self.attacks is not None:- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")+ if not isinstance(self.attacks, list) or not all(+ attack.multi_turn == False for attack in self.attacks+ ):+ raise ValueError(+ "The 'attacks' passed must be a list of single-turn attacks"+ )def _get_turns(self,@@ -179,7 +183,9 @@ class CrescendoJailbreaking(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- current_attack = enhance_attack(attack, current_attack, self.simulator_model)+ current_attack = enhance_attack(+ attack, current_attack, self.simulator_model+ )turns.append(RTTurn(role="user", content=current_attack))@@ -334,7 +340,9 @@ class CrescendoJailbreaking(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- current_attack = await a_enhance_attack(attack, current_attack, self.simulator_model)+ current_attack = await a_enhance_attack(+ attack, current_attack, self.simulator_model+ )turns.append(RTTurn(role="user", content=current_attack))diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.pyindex edbaad06..0f82711e 100644--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py@@ -43,9 +43,12 @@ class LinearJailbreaking(BaseAttack):self.attacks = attacksif self.attacks is not None:- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")-+ if not isinstance(self.attacks, list) or not all(+ attack.multi_turn == False for attack in self.attacks+ ):+ raise ValueError(+ "The 'attacks' passed must be a list of single-turn attacks"+ )def _get_turns(self,@@ -130,7 +133,9 @@ class LinearJailbreaking(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- current_attack = enhance_attack(attack, current_attack, self.simulator_model)+ current_attack = enhance_attack(+ attack, current_attack, self.simulator_model+ )target_response = model_callback(current_attack, turns)turns.append(RTTurn(role="user", content=current_attack))@@ -224,7 +229,9 @@ class LinearJailbreaking(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- current_attack = await a_enhance_attack(attack, current_attack, self.simulator_model)+ current_attack = await a_enhance_attack(+ attack, current_attack, self.simulator_model+ )target_response = await model_callback(current_attack, turns)turns.append(RTTurn(role="user", content=current_attack))diff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.pyindex 8b510e5c..9182f866 100644--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py@@ -48,8 +48,12 @@ class SequentialJailbreak(BaseAttack):self.attacks = attacksif self.attacks is not None:- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")+ if not isinstance(self.attacks, list) or not all(+ attack.multi_turn == False for attack in self.attacks+ ):+ raise ValueError(+ "The 'attacks' passed must be a list of single-turn attacks"+ )# Validate parametersif (@@ -157,7 +161,9 @@ class SequentialJailbreak(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- enhanced_attack = enhance_attack(attack, enhanced_attack, self.simulator_model)+ enhanced_attack = enhance_attack(+ attack, enhanced_attack, self.simulator_model+ )turns.append(RTTurn(role="user", content=current_attack))@@ -323,7 +329,9 @@ class SequentialJailbreak(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- enhanced_attack = await a_enhance_attack(attack, enhanced_attack, self.simulator_model)+ enhanced_attack = await a_enhance_attack(+ attack, enhanced_attack, self.simulator_model+ )turns.append(RTTurn(role="user", content=current_attack))@@ -556,4 +564,4 @@ class SequentialJailbreak(BaseAttack):return resultdef get_name(self) -> str:- return "Sequential Jailbreak"\ No newline at end of file+ return "Sequential Jailbreak"diff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.pyindex 86dd449e..2d49a46e 100644--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py@@ -62,8 +62,12 @@ class TreeJailbreaking(BaseAttack):self.attacks = attacksif self.attacks is not None:- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")+ if not isinstance(self.attacks, list) or not all(+ attack.multi_turn == False for attack in self.attacks+ ):+ raise ValueError(+ "The 'attacks' passed must be a list of single-turn attacks"+ )def _get_turns(self,@@ -338,7 +342,7 @@ class TreeJailbreaking(BaseAttack):vulnerability_data: str = "",):if node.depth >= self.max_depth:- return+ return"""Expands a tree node by generating multiple children nodes based on branching logic."""if node.depth == 0:@@ -352,7 +356,7 @@ class TreeJailbreaking(BaseAttack):]else:conversation_json = node.conversation_history-+conversation_json = list(conversation_json)conversation_json.append({@@ -373,7 +377,9 @@ class TreeJailbreaking(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- enhanced_attack = enhance_attack(attack, enhanced_attack, self.simulator_model)+ enhanced_attack = enhance_attack(+ attack, enhanced_attack, self.simulator_model+ )# Check if enhanced attack is a refusal statementnon_refusal_prompt = JailBreakingTemplate.non_refusal(@@ -530,7 +536,7 @@ class TreeJailbreaking(BaseAttack):vulnerability_data: str = "",):if node.depth >= self.max_depth:- return+ return"""Expands a tree node asynchronously by generating multiple children nodes."""if node.depth == 0:@@ -588,7 +594,9 @@ class TreeJailbreaking(BaseAttack):# Randomly enhancing a turn attackif self.attacks and random.random() < 0.5:attack = random.choice(self.attacks)- enhanced_attack = await a_enhance_attack(attack, enhanced_attack, self.simulator_model)+ enhanced_attack = await a_enhance_attack(+ attack, enhanced_attack, self.simulator_model+ )# Check if enhanced attack is a refusal statementnon_refusal_prompt = JailBreakingTemplate.non_refusal(diff --git a/deepteam/attacks/multi_turn/utils.py b/deepteam/attacks/multi_turn/utils.pyindex 654e6530..33898d7b 100644--- a/deepteam/attacks/multi_turn/utils.py+++ b/deepteam/attacks/multi_turn/utils.py@@ -23,15 +23,14 @@ def update_turn_history(return turn_history+def enhance_attack(- attack: BaseAttack,- current_attack: str,- simulator_model: DeepEvalBaseLLM+ attack: BaseAttack, current_attack: str, simulator_model: DeepEvalBaseLLM):sig = inspect.signature(attack.enhance)try:res = current_attack- if ("simulator_model" in sig.parameters):+ if "simulator_model" in sig.parameters:res = attack.enhance(attack=current_attack,simulator_model=simulator_model,@@ -42,16 +41,15 @@ def enhance_attack(return resexcept:return current_attack-++async def a_enhance_attack(- attack: BaseAttack,- current_attack: str,- simulator_model: DeepEvalBaseLLM+ attack: BaseAttack, current_attack: str, simulator_model: DeepEvalBaseLLM):sig = inspect.signature(attack.enhance)try:res = current_attack- if ("simulator_model" in sig.parameters):+ if "simulator_model" in sig.parameters:res = await attack.a_enhance(attack=current_attack,simulator_model=simulator_model,@@ -61,4 +59,4 @@ async def a_enhance_attack(return resexcept:- return current_attack\ No newline at end of file+ return current_attackdiff --git a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.pyindex 6890fcf7..35224dc3 100644--- a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py+++ b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py@@ -36,9 +36,7 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = AmnesiaAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = AmnesiaAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py b/deepteam/metrics/agentic/boundary_violation/boundary_violation.pyindex 00b737a1..9b7d16f4 100644--- a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py+++ b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py@@ -36,9 +36,7 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BoundaryViolationTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BoundaryViolationTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/domain_validation/domain_validation.py b/deepteam/metrics/agentic/domain_validation/domain_validation.pyindex 295c8be1..84058499 100644--- a/deepteam/metrics/agentic/domain_validation/domain_validation.py+++ b/deepteam/metrics/agentic/domain_validation/domain_validation.py@@ -36,9 +36,7 @@ class DomainValidationMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = DomainValidationTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class DomainValidationMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = DomainValidationTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/escalation_success/escalation_success.py b/deepteam/metrics/agentic/escalation_success/escalation_success.pyindex 43117c89..9c12b6a8 100644--- a/deepteam/metrics/agentic/escalation_success/escalation_success.py+++ b/deepteam/metrics/agentic/escalation_success/escalation_success.py@@ -36,9 +36,7 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = EscalationSuccessTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = EscalationSuccessTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/extraction_success/extraction_success.py b/deepteam/metrics/agentic/extraction_success/extraction_success.pyindex 6b9c2a18..9c496597 100644--- a/deepteam/metrics/agentic/extraction_success/extraction_success.py+++ b/deepteam/metrics/agentic/extraction_success/extraction_success.py@@ -36,9 +36,7 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ExtractionSuccessTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ExtractionSuccessTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/goal_drift/goal_drift.py b/deepteam/metrics/agentic/goal_drift/goal_drift.pyindex 0c21d454..d35c5c0d 100644--- a/deepteam/metrics/agentic/goal_drift/goal_drift.py+++ b/deepteam/metrics/agentic/goal_drift/goal_drift.py@@ -36,9 +36,7 @@ class GoalDriftMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = GoalDriftTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class GoalDriftMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = GoalDriftTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.pyindex 907e1a6a..849383ff 100644--- a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py+++ b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py@@ -36,9 +36,7 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HallucinationDetectionTemplate.generate_evaluation_results(actual_output=actual_output,@@ -157,7 +159,11 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HallucinationDetectionTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.pyindex fc5ab68e..40cd055f 100644--- a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py+++ b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py@@ -36,9 +36,7 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HierarchyConsistencyTemplate.generate_evaluation_results(actual_output=actual_output,@@ -157,7 +159,11 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HierarchyConsistencyTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py b/deepteam/metrics/agentic/leakage_rate/leakage_rate.pyindex bf50ec5c..20dba03c 100644--- a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py+++ b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py@@ -36,9 +36,7 @@ class LeakageRateMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = LeakageRateTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class LeakageRateMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = LeakageRateTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.pyindex b7ec7265..4aba38d6 100644--- a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py+++ b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py@@ -36,9 +36,7 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ManipulationAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,@@ -157,7 +159,11 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ManipulationAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py b/deepteam/metrics/agentic/misinterpretation/misinterpretation.pyindex caddc33d..5b33c7d0 100644--- a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py+++ b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py@@ -36,9 +36,7 @@ class MisinterpretationMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = MisinterpretationTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class MisinterpretationMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = MisinterpretationTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.pyindex 78c6f968..4b54436f 100644--- a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py+++ b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py@@ -36,9 +36,7 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = PoisoningAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = PoisoningAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/subversion_success/subversion_success.py b/deepteam/metrics/agentic/subversion_success/subversion_success.pyindex f4000bf7..93ee818d 100644--- a/deepteam/metrics/agentic/subversion_success/subversion_success.py+++ b/deepteam/metrics/agentic/subversion_success/subversion_success.py@@ -36,9 +36,7 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SubversionSuccessTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SubversionSuccessTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.pyindex b421b8f7..8b96c9a6 100644--- a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py+++ b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py@@ -36,9 +36,7 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -121,11 +119,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = TemporalAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,@@ -153,7 +155,11 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = TemporalAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.pyindex a660f877..397343e7 100644--- a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py+++ b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py@@ -36,9 +36,7 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = UnauthorizedExecutionTemplate.generate_evaluation_results(actual_output=actual_output,@@ -157,7 +159,11 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = UnauthorizedExecutionTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py b/deepteam/metrics/agentic/verification_assessment/verification_assessment.pyindex 200499ef..58153885 100644--- a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py+++ b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py@@ -36,9 +36,7 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = VerificationAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,@@ -157,7 +159,11 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = VerificationAssessmentTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/bfla/bfla.py b/deepteam/metrics/bfla/bfla.pyindex 9d98c761..f01a4c2a 100644--- a/deepteam/metrics/bfla/bfla.py+++ b/deepteam/metrics/bfla/bfla.py@@ -34,9 +34,7 @@ class BFLAMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class BFLAMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BFLATemplate.generate_evaluation_results(input=test_case.input,@@ -158,7 +160,11 @@ class BFLAMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BFLATemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/bias/bias.py b/deepteam/metrics/bias/bias.pyindex 99a69930..d350a330 100644--- a/deepteam/metrics/bias/bias.py+++ b/deepteam/metrics/bias/bias.py@@ -17,6 +17,7 @@ from deepteam.metrics.bias.template import (from deepteam.metrics.bias.schema import *from deepteam.utils import format_turns+class BiasMetric(BaseRedTeamingMetric):def __init__(self,@@ -35,9 +36,7 @@ class BiasMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -120,11 +119,15 @@ class BiasMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BiasTemplate.generate_evaluation_results(actual_output=actual_output,@@ -152,7 +155,11 @@ class BiasMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BiasTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/bola/bola.py b/deepteam/metrics/bola/bola.pyindex acc73d65..b0507be2 100644--- a/deepteam/metrics/bola/bola.py+++ b/deepteam/metrics/bola/bola.py@@ -34,9 +34,7 @@ class BOLAMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -123,11 +121,15 @@ class BOLAMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BOLATemplate.generate_evaluation_results(input=test_case.input,@@ -156,7 +158,11 @@ class BOLAMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = BOLATemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/competitors/competitors.py b/deepteam/metrics/competitors/competitors.pyindex 7a84ee28..f0aa43db 100644--- a/deepteam/metrics/competitors/competitors.py+++ b/deepteam/metrics/competitors/competitors.py@@ -34,9 +34,7 @@ class CompetitorsMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class CompetitorsMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = CompetitorsTemplate.generate_evaluation_results(input=test_case.input,@@ -158,7 +160,11 @@ class CompetitorsMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = CompetitorsTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/contracts/contracts.py b/deepteam/metrics/contracts/contracts.pyindex 2efb0aa3..8f0ef15e 100644--- a/deepteam/metrics/contracts/contracts.py+++ b/deepteam/metrics/contracts/contracts.py@@ -30,9 +30,7 @@ class ContractsMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -81,11 +79,15 @@ class ContractsMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ContractsTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_output@@ -111,7 +113,11 @@ class ContractsMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ContractsTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_outputdiff --git a/deepteam/metrics/debug_access/debug_access.py b/deepteam/metrics/debug_access/debug_access.pyindex 60a143a4..0a78dd72 100644--- a/deepteam/metrics/debug_access/debug_access.py+++ b/deepteam/metrics/debug_access/debug_access.py@@ -30,9 +30,7 @@ class DebugAccessMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -81,11 +79,15 @@ class DebugAccessMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = DebugAccessTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_output@@ -112,7 +114,11 @@ class DebugAccessMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = DebugAccessTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_outputdiff --git a/deepteam/metrics/excessive_agency/excessive_agency.py b/deepteam/metrics/excessive_agency/excessive_agency.pyindex 632a5a75..8abda633 100644--- a/deepteam/metrics/excessive_agency/excessive_agency.py+++ b/deepteam/metrics/excessive_agency/excessive_agency.py@@ -32,9 +32,7 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):self.verbose_mode = verbose_modeself.purpose = purpose- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -83,11 +81,15 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ExcessiveAgencyTemplate.generate_evaluation_results(input=test_case.input,@@ -116,7 +118,11 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ExcessiveAgencyTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/graphic_content/graphic_content.py b/deepteam/metrics/graphic_content/graphic_content.pyindex 11955bc2..0714c23b 100644--- a/deepteam/metrics/graphic_content/graphic_content.py+++ b/deepteam/metrics/graphic_content/graphic_content.py@@ -31,9 +31,7 @@ class GraphicMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -82,11 +80,15 @@ class GraphicMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = GraphicTemplate.generate_evaluation_results(input=test_case.input,@@ -115,7 +117,11 @@ class GraphicMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = GraphicTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/hallucination/hallucination.py b/deepteam/metrics/hallucination/hallucination.pyindex 99a4413f..19145c09 100644--- a/deepteam/metrics/hallucination/hallucination.py+++ b/deepteam/metrics/hallucination/hallucination.py@@ -34,9 +34,7 @@ class HallucinationMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class HallucinationMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HallucinationTemplate.generate_evaluation_results(actual_output=actual_output,@@ -157,7 +159,11 @@ class HallucinationMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HallucinationTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/harm/harm.py b/deepteam/metrics/harm/harm.pyindex 627ec0e2..21cab270 100644--- a/deepteam/metrics/harm/harm.py+++ b/deepteam/metrics/harm/harm.py@@ -31,9 +31,7 @@ class HarmMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -82,11 +80,15 @@ class HarmMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HarmTemplate.generate_evaluation_results(input=test_case.input,@@ -115,7 +117,11 @@ class HarmMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HarmTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/hijacking/hijacking.py b/deepteam/metrics/hijacking/hijacking.pyindex 80027162..f190711e 100644--- a/deepteam/metrics/hijacking/hijacking.py+++ b/deepteam/metrics/hijacking/hijacking.py@@ -32,9 +32,7 @@ class HijackingMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -123,11 +121,15 @@ class HijackingMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HijackingTemplate.generate_evaluation_results(input=test_case.input,@@ -156,7 +158,11 @@ class HijackingMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = HijackingTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/illegal_activity/illegal_activity.py b/deepteam/metrics/illegal_activity/illegal_activity.pyindex 93c97b5b..c425c470 100644--- a/deepteam/metrics/illegal_activity/illegal_activity.py+++ b/deepteam/metrics/illegal_activity/illegal_activity.py@@ -31,9 +31,7 @@ class IllegalMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -82,11 +80,15 @@ class IllegalMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = IllegalTemplate.generate_evaluation_results(input=test_case.input,@@ -115,7 +117,11 @@ class IllegalMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = IllegalTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/imitation/imitation.py b/deepteam/metrics/imitation/imitation.pyindex 4ee7a3e4..86c52df9 100644--- a/deepteam/metrics/imitation/imitation.py+++ b/deepteam/metrics/imitation/imitation.py@@ -34,9 +34,7 @@ class ImitationMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -123,11 +121,15 @@ class ImitationMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ImitationTemplate.generate_evaluation_results(input=test_case.input,@@ -156,7 +158,11 @@ class ImitationMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ImitationTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/intellectual_property/intellectual_property.py b/deepteam/metrics/intellectual_property/intellectual_property.pyindex 94962180..20099484 100644--- a/deepteam/metrics/intellectual_property/intellectual_property.py+++ b/deepteam/metrics/intellectual_property/intellectual_property.py@@ -32,9 +32,7 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -127,11 +125,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = IntellectualPropertyTemplate.generate_evaluation_results(actual_output=actual_output,@@ -159,7 +161,11 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = IntellectualPropertyTemplate.generate_evaluation_results(actual_output=actual_output,diff --git a/deepteam/metrics/misinformation/misinformation.py b/deepteam/metrics/misinformation/misinformation.pyindex 81c1d854..52098f2a 100644--- a/deepteam/metrics/misinformation/misinformation.py+++ b/deepteam/metrics/misinformation/misinformation.py@@ -31,9 +31,7 @@ class MisinformationMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -82,11 +80,15 @@ class MisinformationMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = MisinformationTemplate.generate_evaluation_results(input=test_case.input,@@ -115,7 +117,11 @@ class MisinformationMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = MisinformationTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/overreliance/overreliance.py b/deepteam/metrics/overreliance/overreliance.pyindex f0bcf8e9..5f390b2c 100644--- a/deepteam/metrics/overreliance/overreliance.py+++ b/deepteam/metrics/overreliance/overreliance.py@@ -34,9 +34,7 @@ class OverrelianceMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -125,11 +123,15 @@ class OverrelianceMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = OverrelianceTemplate.generate_evaluation_results(input=test_case.input,@@ -158,7 +160,11 @@ class OverrelianceMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = OverrelianceTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/personal_safety/personal_safety.py b/deepteam/metrics/personal_safety/personal_safety.pyindex c7ae17b2..584c37d8 100644--- a/deepteam/metrics/personal_safety/personal_safety.py+++ b/deepteam/metrics/personal_safety/personal_safety.py@@ -33,9 +33,7 @@ class SafetyMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -84,11 +82,15 @@ class SafetyMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SafetyTemplate.generate_evaluation_results(input=test_case.input,@@ -117,7 +119,11 @@ class SafetyMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SafetyTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/pii/pii.py b/deepteam/metrics/pii/pii.pyindex f93a7c80..2940b11a 100644--- a/deepteam/metrics/pii/pii.py+++ b/deepteam/metrics/pii/pii.py@@ -34,9 +34,7 @@ class PIIMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -165,11 +163,15 @@ class PIIMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = PIITemplate.generate_evaluation_results(input=test_case.input,@@ -199,7 +201,11 @@ class PIIMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = PIITemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/prompt_extraction/prompt_extraction.py b/deepteam/metrics/prompt_extraction/prompt_extraction.pyindex b0a5be32..d5c6f305 100644--- a/deepteam/metrics/prompt_extraction/prompt_extraction.py+++ b/deepteam/metrics/prompt_extraction/prompt_extraction.py@@ -32,9 +32,7 @@ class PromptExtractionMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -123,11 +121,15 @@ class PromptExtractionMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = PromptExtractionTemplate.generate_evaluation_results(input=test_case.input,@@ -156,7 +158,11 @@ class PromptExtractionMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = PromptExtractionTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/rbac/rbac.py b/deepteam/metrics/rbac/rbac.pyindex 26498b05..3915a1ec 100644--- a/deepteam/metrics/rbac/rbac.py+++ b/deepteam/metrics/rbac/rbac.py@@ -34,9 +34,7 @@ class RBACMetric(BaseRedTeamingMetric):except:self.system_prompt = ""- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -87,12 +85,16 @@ class RBACMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"-+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )+prompt = RBACTemplate.generate_evaluation_results(input=test_case.input,actual_output=actual_output,@@ -120,7 +122,11 @@ class RBACMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = RBACTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/shell_injection/shell_injection.py b/deepteam/metrics/shell_injection/shell_injection.pyindex d0ba2afc..60d2735d 100644--- a/deepteam/metrics/shell_injection/shell_injection.py+++ b/deepteam/metrics/shell_injection/shell_injection.py@@ -17,6 +17,7 @@ from deepteam.metrics.shell_injection.template import ()from deepteam.utils import format_turns+class ShellInjectionMetric(BaseRedTeamingMetric):def __init__(self,@@ -29,9 +30,7 @@ class ShellInjectionMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -80,11 +79,15 @@ class ShellInjectionMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ShellInjectionTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_output@@ -111,7 +114,11 @@ class ShellInjectionMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ShellInjectionTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_outputdiff --git a/deepteam/metrics/sql_injection/sql_injection.py b/deepteam/metrics/sql_injection/sql_injection.pyindex d355af08..6e79c4ca 100644--- a/deepteam/metrics/sql_injection/sql_injection.py+++ b/deepteam/metrics/sql_injection/sql_injection.py@@ -30,9 +30,7 @@ class SQLInjectionMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -79,11 +77,15 @@ class SQLInjectionMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SQLInjectionTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_output@@ -110,7 +112,11 @@ class SQLInjectionMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SQLInjectionTemplate.generate_evaluation_results(input=test_case.input, actual_output=actual_outputdiff --git a/deepteam/metrics/ssrf/ssrf.py b/deepteam/metrics/ssrf/ssrf.pyindex 9239e8c0..8a0e5fcb 100644--- a/deepteam/metrics/ssrf/ssrf.py+++ b/deepteam/metrics/ssrf/ssrf.py@@ -19,6 +19,7 @@ from deepteam.metrics.ssrf.template import ()from deepteam.utils import format_turns+class SSRFMetric(BaseRedTeamingMetric):def __init__(self,@@ -33,9 +34,7 @@ class SSRFMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -124,11 +123,15 @@ class SSRFMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SSRFTemplate.generate_evaluation_results(input=test_case.input,@@ -157,7 +160,11 @@ class SSRFMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = SSRFTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/metrics/toxicity/toxicity.py b/deepteam/metrics/toxicity/toxicity.pyindex 19d5f55c..4bb22a42 100644--- a/deepteam/metrics/toxicity/toxicity.py+++ b/deepteam/metrics/toxicity/toxicity.py@@ -31,9 +31,7 @@ class ToxicityMetric(BaseRedTeamingMetric):self.async_mode = async_modeself.verbose_mode = verbose_mode- def measure(- self, test_case: RTTestCase- ) -> float:+ def measure(self, test_case: RTTestCase) -> float:self.evaluation_cost = 0 if self.using_native_model else Nonewith metric_progress_indicator(self, _show_indicator=False):@@ -82,11 +80,15 @@ class ToxicityMetric(BaseRedTeamingMetric):async def _a_evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-+if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ToxicityTemplate.generate_evaluation_results(input=test_case.input,@@ -115,7 +117,11 @@ class ToxicityMetric(BaseRedTeamingMetric):if test_case.turns is not None:actual_output = format_turns(test_case.turns)else:- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"+ actual_output = (+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"+ + test_case.actual_output+ + "\n"+ )prompt = ToxicityTemplate.generate_evaluation_results(input=test_case.input,diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex d9ac816b..7e364c3d 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -106,9 +106,7 @@ class RedTeamer:and self.test_cases is not Noneand len(self.test_cases) > 0):- simulated_test_cases: List[RTTestCase] = (- self.test_cases- )+ simulated_test_cases: List[RTTestCase] = self.test_caseselse:self.attack_simulator.model_callback = model_callbacksimulated_test_cases: List[RTTestCase] = (@@ -210,9 +208,7 @@ class RedTeamer:and self.test_cases is not Noneand len(self.test_cases) > 0):- simulated_test_cases: List[RTTestCase] = (- self.test_cases- )+ simulated_test_cases: List[RTTestCase] = self.test_caseselse:self.attack_simulator.model_callback = model_callbacksimulated_test_cases: List[RTTestCase] = (@@ -417,7 +413,9 @@ class RedTeamer:simulated_test_case.input, simulated_test_case.turns)else:- actual_output = await model_callback(simulated_test_case.input)+ actual_output = await model_callback(+ simulated_test_case.input+ )except Exception:if ignore_errors:red_teaming_test_case.error = (diff --git a/deepteam/red_teamer/risk_assessment.py b/deepteam/red_teamer/risk_assessment.pyindex c1db9ff6..48473b43 100644--- a/deepteam/red_teamer/risk_assessment.py+++ b/deepteam/red_teamer/risk_assessment.py@@ -93,9 +93,9 @@ class RiskAssessment(BaseModel):def __init__(self, **data):super().__init__(**data)- self.test_cases: TestCasesList = TestCasesList[- RTTestCase- ](self.test_cases)+ self.test_cases: TestCasesList = TestCasesList[RTTestCase](+ self.test_cases+ )def save(self, to: str) -> str:try:@@ -134,9 +134,7 @@ def construct_risk_assessment_overview(VulnerabilityType,List[RTTestCase],] = {}- attack_method_to_cases: Dict[- str, List[RTTestCase]- ] = {}+ attack_method_to_cases: Dict[str, List[RTTestCase]] = {}errored = 0for test_case in red_teaming_test_cases:diff --git a/deepteam/test_case/__init__.py b/deepteam/test_case/__init__.pyindex 4822859d..e66dffcb 100644--- a/deepteam/test_case/__init__.py+++ b/deepteam/test_case/__init__.py@@ -1 +1 @@-from .test_case import RTTestCase, RTTurn\ No newline at end of file+from .test_case import RTTestCase, RTTurndiff --git a/deepteam/test_case/test_case.py b/deepteam/test_case/test_case.pyindex b0485939..f54c4e36 100644--- a/deepteam/test_case/test_case.py+++ b/deepteam/test_case/test_case.py@@ -7,6 +7,7 @@ from deepeval.test_case import LLMTestCase, Turnclass RTTurn(Turn):pass+class RTTestCase(LLMTestCase):vulnerability: strinput: Optional[str] = None@@ -41,12 +42,14 @@ class RTTestCase(LLMTestCase):if actual_output is not None:if not isinstance(actual_output, str):raise TypeError("'actual_output' must be a string")-+if turns is not None:- if not isinstance(turns, list) or not all(isinstance(turn, RTTurn) for turn in turns):+ if not isinstance(turns, list) or not all(+ isinstance(turn, RTTurn) for turn in turns+ ):raise TypeError("'turns' must be a list of 'RTTurn'")-- if actual_output is not None and turns is not None:++ if actual_output is not None and turns is not None:raise ValueError("An 'RTTestCase' cannot contain both 'actual_output' and 'turns' at the same time.")@@ -54,33 +57,33 @@ class RTTestCase(LLMTestCase):if vulnerability is not None:if not isinstance(vulnerability, str):raise TypeError("'vulnerability' must be a string")-+if vulnerability_type is not None:if not isinstance(vulnerability, str):raise TypeError("'vulnerability_type' must be an Enum")-+if metadata is not None:if not isinstance(metadata, dict):raise TypeError("'metadata' must be a dictionary")-+if attack_method is not None:if not isinstance(attack_method, str):raise TypeError("'attack_method' must be a string")-+if risk_category is not None:if not isinstance(risk_category, str):raise TypeError("'risk_category' must be a string")-+if score is not None:if not isinstance(score, float):raise TypeError("'score' must be a float")-+if reason is not None:if not isinstance(reason, str):raise TypeError("'reason' must be a string")-+if error is not None:if not isinstance(error, str):raise TypeError("'error' must be a string")-- return data\ No newline at end of file++ return datadiff --git a/deepteam/utils.py b/deepteam/utils.pyindex 38f2d363..bbe15fa4 100644--- a/deepteam/utils.py+++ b/deepteam/utils.py@@ -16,14 +16,15 @@ def validate_model_callback_signature("`model_callback` must not be async. `async_mode` has been set to False.")+def format_turns(turns: List[RTTurn]):if not turns:raise ValueError("There are no 'turns' to format.")-+formatted_turns = "Full Conversation To Evaluate: \n"for turn in turns:formatted_turns += f"Role: {turn.role} \n"formatted_turns += f"Content: {turn.content} \n\n"formatted_turns += "End of conversation. \n"- return formatted_turns\ No newline at end of file+ return formatted_turnsdiff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex 7bd7cf62..71a911a1 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -179,7 +179,7 @@ class CustomVulnerability(BaseVulnerability):purpose: str = None,attacks_per_vulnerability_type: int = 1,) -> List[RTTestCase]:-+self.purpose = purposeself.simulator_model, self.using_native_model = initialize_model(@@ -197,7 +197,7 @@ class CustomVulnerability(BaseVulnerability):type,attacks_per_vulnerability_type,self.custom_prompt,- self.purpose+ self.purpose,))@@ -237,7 +237,7 @@ class CustomVulnerability(BaseVulnerability):purpose: str = None,attacks_per_vulnerability_type: int = 1,) -> List[RTTestCase]:-+self.purpose = purposeself.simulator_model, self.using_native_model = initialize_model(@@ -255,7 +255,7 @@ class CustomVulnerability(BaseVulnerability):type,attacks_per_vulnerability_type,self.custom_prompt,- self.purpose+ self.purpose,))diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex 22327f39..e95d1faa 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -79,9 +79,7 @@ class IntellectualProperty(BaseVulnerability):purpose, attacks_per_vulnerability_type)- results: Dict[IntellectualPropertyType, List[RTTestCase]] = (- dict()- )+ results: Dict[IntellectualPropertyType, List[RTTestCase]] = dict()for simulated_test_case in simulated_test_cases:vulnerability_type = simulated_test_case.vulnerability_type@@ -123,9 +121,7 @@ class IntellectualProperty(BaseVulnerability):purpose, attacks_per_vulnerability_type)- results: Dict[IntellectualPropertyType, List[RTTestCase]] = (- dict()- )+ results: Dict[IntellectualPropertyType, List[RTTestCase]] = dict()async def process_attack(simulated_test_case: RTTestCase):vulnerability_type = simulated_test_case.vulnerability_type