Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

holy reformat

penguine06b51994e408
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.py
index 50e6cb9f..2c4b0c38 100644
--- a/deepteam/attacks/attack_simulator/attack_simulator.py
+++ b/deepteam/attacks/attack_simulator/attack_simulator.py
@@ -170,9 +170,7 @@ class AttackSimulator:
enhanced_test_cases.extend(
await asyncio.gather(
*[
- asyncio.create_task(
- throttled_attack_method(test_case)
- )
+ asyncio.create_task(throttled_attack_method(test_case))
for test_case in test_cases
]
)
@@ -346,9 +344,7 @@ class AttackSimulator:
raise
except Exception as e:
if ignore_errors:
- test_case.error = (
- f"Error enhancing regular attack: {str(e)}"
- )
+ test_case.error = f"Error enhancing regular attack: {str(e)}"
return test_case
else:
raise
diff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
index f26ed4d1..4122afa0 100644
--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
@@ -43,8 +43,12 @@ class BadLikertJudge(BaseAttack):
self.attacks = attacks
if self.attacks is not None:
- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):
- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")
+ if not isinstance(self.attacks, list) or not all(
+ attack.multi_turn == False for attack in self.attacks
+ ):
+ raise ValueError(
+ "The 'attacks' passed must be a list of single-turn attacks"
+ )
def _get_turns(
self,
@@ -140,7 +144,9 @@ class BadLikertJudge(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- enhanced_attack = enhance_attack(attack, enhanced_attack, self.simulator_model)
+ enhanced_attack = enhance_attack(
+ attack, enhanced_attack, self.simulator_model
+ )
assistant_response = model_callback(current_attack, turns)
turns.append(RTTurn(role="user", content=current_attack))
@@ -246,7 +252,9 @@ class BadLikertJudge(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- enhanced_attack = await a_enhance_attack(attack, enhanced_attack, self.simulator_model)
+ enhanced_attack = await a_enhance_attack(
+ attack, enhanced_attack, self.simulator_model
+ )
assistant_response = await model_callback(current_attack, turns)
turns.append(RTTurn(role="user", content=current_attack))
diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
index e01433c0..e6d1ed0e 100644
--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
@@ -71,8 +71,12 @@ class CrescendoJailbreaking(BaseAttack):
self.attacks = attacks
if self.attacks is not None:
- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):
- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")
+ if not isinstance(self.attacks, list) or not all(
+ attack.multi_turn == False for attack in self.attacks
+ ):
+ raise ValueError(
+ "The 'attacks' passed must be a list of single-turn attacks"
+ )
def _get_turns(
self,
@@ -179,7 +183,9 @@ class CrescendoJailbreaking(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- current_attack = enhance_attack(attack, current_attack, self.simulator_model)
+ current_attack = enhance_attack(
+ attack, current_attack, self.simulator_model
+ )
turns.append(RTTurn(role="user", content=current_attack))
@@ -334,7 +340,9 @@ class CrescendoJailbreaking(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- current_attack = await a_enhance_attack(attack, current_attack, self.simulator_model)
+ current_attack = await a_enhance_attack(
+ attack, current_attack, self.simulator_model
+ )
turns.append(RTTurn(role="user", content=current_attack))
diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
index edbaad06..0f82711e 100644
--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
@@ -43,9 +43,12 @@ class LinearJailbreaking(BaseAttack):
self.attacks = attacks
if self.attacks is not None:
- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):
- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")
-
+ if not isinstance(self.attacks, list) or not all(
+ attack.multi_turn == False for attack in self.attacks
+ ):
+ raise ValueError(
+ "The 'attacks' passed must be a list of single-turn attacks"
+ )
def _get_turns(
self,
@@ -130,7 +133,9 @@ class LinearJailbreaking(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- current_attack = enhance_attack(attack, current_attack, self.simulator_model)
+ current_attack = enhance_attack(
+ attack, current_attack, self.simulator_model
+ )
target_response = model_callback(current_attack, turns)
turns.append(RTTurn(role="user", content=current_attack))
@@ -224,7 +229,9 @@ class LinearJailbreaking(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- current_attack = await a_enhance_attack(attack, current_attack, self.simulator_model)
+ current_attack = await a_enhance_attack(
+ attack, current_attack, self.simulator_model
+ )
target_response = await model_callback(current_attack, turns)
turns.append(RTTurn(role="user", content=current_attack))
diff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
index 8b510e5c..9182f866 100644
--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
@@ -48,8 +48,12 @@ class SequentialJailbreak(BaseAttack):
self.attacks = attacks
if self.attacks is not None:
- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):
- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")
+ if not isinstance(self.attacks, list) or not all(
+ attack.multi_turn == False for attack in self.attacks
+ ):
+ raise ValueError(
+ "The 'attacks' passed must be a list of single-turn attacks"
+ )
# Validate parameters
if (
@@ -157,7 +161,9 @@ class SequentialJailbreak(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- enhanced_attack = enhance_attack(attack, enhanced_attack, self.simulator_model)
+ enhanced_attack = enhance_attack(
+ attack, enhanced_attack, self.simulator_model
+ )
turns.append(RTTurn(role="user", content=current_attack))
@@ -323,7 +329,9 @@ class SequentialJailbreak(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- enhanced_attack = await a_enhance_attack(attack, enhanced_attack, self.simulator_model)
+ enhanced_attack = await a_enhance_attack(
+ attack, enhanced_attack, self.simulator_model
+ )
turns.append(RTTurn(role="user", content=current_attack))
@@ -556,4 +564,4 @@ class SequentialJailbreak(BaseAttack):
return result
def get_name(self) -> str:
- return "Sequential Jailbreak"
\ No newline at end of file
+ return "Sequential Jailbreak"
diff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
index 86dd449e..2d49a46e 100644
--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
@@ -62,8 +62,12 @@ class TreeJailbreaking(BaseAttack):
self.attacks = attacks
if self.attacks is not None:
- if not isinstance(self.attacks, list) or not all(attack.multi_turn == False for attack in self.attacks):
- raise ValueError("The 'attacks' passed must be a list of single-turn attacks")
+ if not isinstance(self.attacks, list) or not all(
+ attack.multi_turn == False for attack in self.attacks
+ ):
+ raise ValueError(
+ "The 'attacks' passed must be a list of single-turn attacks"
+ )
def _get_turns(
self,
@@ -338,7 +342,7 @@ class TreeJailbreaking(BaseAttack):
vulnerability_data: str = "",
):
if node.depth >= self.max_depth:
- return
+ return
"""Expands a tree node by generating multiple children nodes based on branching logic."""
if node.depth == 0:
@@ -352,7 +356,7 @@ class TreeJailbreaking(BaseAttack):
]
else:
conversation_json = node.conversation_history
-
+
conversation_json = list(conversation_json)
conversation_json.append(
{
@@ -373,7 +377,9 @@ class TreeJailbreaking(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- enhanced_attack = enhance_attack(attack, enhanced_attack, self.simulator_model)
+ enhanced_attack = enhance_attack(
+ attack, enhanced_attack, self.simulator_model
+ )
# Check if enhanced attack is a refusal statement
non_refusal_prompt = JailBreakingTemplate.non_refusal(
@@ -530,7 +536,7 @@ class TreeJailbreaking(BaseAttack):
vulnerability_data: str = "",
):
if node.depth >= self.max_depth:
- return
+ return
"""Expands a tree node asynchronously by generating multiple children nodes."""
if node.depth == 0:
@@ -588,7 +594,9 @@ class TreeJailbreaking(BaseAttack):
# Randomly enhancing a turn attack
if self.attacks and random.random() < 0.5:
attack = random.choice(self.attacks)
- enhanced_attack = await a_enhance_attack(attack, enhanced_attack, self.simulator_model)
+ enhanced_attack = await a_enhance_attack(
+ attack, enhanced_attack, self.simulator_model
+ )
# Check if enhanced attack is a refusal statement
non_refusal_prompt = JailBreakingTemplate.non_refusal(
diff --git a/deepteam/attacks/multi_turn/utils.py b/deepteam/attacks/multi_turn/utils.py
index 654e6530..33898d7b 100644
--- a/deepteam/attacks/multi_turn/utils.py
+++ b/deepteam/attacks/multi_turn/utils.py
@@ -23,15 +23,14 @@ def update_turn_history(
return turn_history
+
def enhance_attack(
- attack: BaseAttack,
- current_attack: str,
- simulator_model: DeepEvalBaseLLM
+ attack: BaseAttack, current_attack: str, simulator_model: DeepEvalBaseLLM
):
sig = inspect.signature(attack.enhance)
try:
res = current_attack
- if ("simulator_model" in sig.parameters):
+ if "simulator_model" in sig.parameters:
res = attack.enhance(
attack=current_attack,
simulator_model=simulator_model,
@@ -42,16 +41,15 @@ def enhance_attack(
return res
except:
return current_attack
-
+
+
async def a_enhance_attack(
- attack: BaseAttack,
- current_attack: str,
- simulator_model: DeepEvalBaseLLM
+ attack: BaseAttack, current_attack: str, simulator_model: DeepEvalBaseLLM
):
sig = inspect.signature(attack.enhance)
try:
res = current_attack
- if ("simulator_model" in sig.parameters):
+ if "simulator_model" in sig.parameters:
res = await attack.a_enhance(
attack=current_attack,
simulator_model=simulator_model,
@@ -61,4 +59,4 @@ async def a_enhance_attack(
return res
except:
- return current_attack
\ No newline at end of file
+ return current_attack
diff --git a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
index 6890fcf7..35224dc3 100644
--- a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
+++ b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
@@ -36,9 +36,7 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = AmnesiaAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = AmnesiaAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
index 00b737a1..9b7d16f4 100644
--- a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
+++ b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
@@ -36,9 +36,7 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BoundaryViolationTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BoundaryViolationTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/domain_validation/domain_validation.py b/deepteam/metrics/agentic/domain_validation/domain_validation.py
index 295c8be1..84058499 100644
--- a/deepteam/metrics/agentic/domain_validation/domain_validation.py
+++ b/deepteam/metrics/agentic/domain_validation/domain_validation.py
@@ -36,9 +36,7 @@ class DomainValidationMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = DomainValidationTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class DomainValidationMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = DomainValidationTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/escalation_success/escalation_success.py b/deepteam/metrics/agentic/escalation_success/escalation_success.py
index 43117c89..9c12b6a8 100644
--- a/deepteam/metrics/agentic/escalation_success/escalation_success.py
+++ b/deepteam/metrics/agentic/escalation_success/escalation_success.py
@@ -36,9 +36,7 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = EscalationSuccessTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = EscalationSuccessTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/extraction_success/extraction_success.py b/deepteam/metrics/agentic/extraction_success/extraction_success.py
index 6b9c2a18..9c496597 100644
--- a/deepteam/metrics/agentic/extraction_success/extraction_success.py
+++ b/deepteam/metrics/agentic/extraction_success/extraction_success.py
@@ -36,9 +36,7 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ExtractionSuccessTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ExtractionSuccessTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/goal_drift/goal_drift.py b/deepteam/metrics/agentic/goal_drift/goal_drift.py
index 0c21d454..d35c5c0d 100644
--- a/deepteam/metrics/agentic/goal_drift/goal_drift.py
+++ b/deepteam/metrics/agentic/goal_drift/goal_drift.py
@@ -36,9 +36,7 @@ class GoalDriftMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = GoalDriftTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class GoalDriftMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = GoalDriftTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
index 907e1a6a..849383ff 100644
--- a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
+++ b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
@@ -36,9 +36,7 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HallucinationDetectionTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -157,7 +159,11 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HallucinationDetectionTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
index fc5ab68e..40cd055f 100644
--- a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
+++ b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
@@ -36,9 +36,7 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HierarchyConsistencyTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -157,7 +159,11 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HierarchyConsistencyTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
index bf50ec5c..20dba03c 100644
--- a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
+++ b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
@@ -36,9 +36,7 @@ class LeakageRateMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = LeakageRateTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class LeakageRateMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = LeakageRateTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
index b7ec7265..4aba38d6 100644
--- a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
+++ b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
@@ -36,9 +36,7 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ManipulationAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -157,7 +159,11 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ManipulationAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
index caddc33d..5b33c7d0 100644
--- a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
+++ b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
@@ -36,9 +36,7 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = MisinterpretationTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = MisinterpretationTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
index 78c6f968..4b54436f 100644
--- a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
+++ b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
@@ -36,9 +36,7 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = PoisoningAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = PoisoningAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/subversion_success/subversion_success.py b/deepteam/metrics/agentic/subversion_success/subversion_success.py
index f4000bf7..93ee818d 100644
--- a/deepteam/metrics/agentic/subversion_success/subversion_success.py
+++ b/deepteam/metrics/agentic/subversion_success/subversion_success.py
@@ -36,9 +36,7 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SubversionSuccessTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SubversionSuccessTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
index b421b8f7..8b96c9a6 100644
--- a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
+++ b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
@@ -36,9 +36,7 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -121,11 +119,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = TemporalAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -153,7 +155,11 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = TemporalAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
index a660f877..397343e7 100644
--- a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
+++ b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
@@ -36,9 +36,7 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = UnauthorizedExecutionTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -157,7 +159,11 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = UnauthorizedExecutionTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
index 200499ef..58153885 100644
--- a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
+++ b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
@@ -36,9 +36,7 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = VerificationAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -157,7 +159,11 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = VerificationAssessmentTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/bfla/bfla.py b/deepteam/metrics/bfla/bfla.py
index 9d98c761..f01a4c2a 100644
--- a/deepteam/metrics/bfla/bfla.py
+++ b/deepteam/metrics/bfla/bfla.py
@@ -34,9 +34,7 @@ class BFLAMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class BFLAMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BFLATemplate.generate_evaluation_results(
input=test_case.input,
@@ -158,7 +160,11 @@ class BFLAMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BFLATemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/bias/bias.py b/deepteam/metrics/bias/bias.py
index 99a69930..d350a330 100644
--- a/deepteam/metrics/bias/bias.py
+++ b/deepteam/metrics/bias/bias.py
@@ -17,6 +17,7 @@ from deepteam.metrics.bias.template import (
from deepteam.metrics.bias.schema import *
from deepteam.utils import format_turns
+
class BiasMetric(BaseRedTeamingMetric):
def __init__(
self,
@@ -35,9 +36,7 @@ class BiasMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -120,11 +119,15 @@ class BiasMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BiasTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -152,7 +155,11 @@ class BiasMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BiasTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/bola/bola.py b/deepteam/metrics/bola/bola.py
index acc73d65..b0507be2 100644
--- a/deepteam/metrics/bola/bola.py
+++ b/deepteam/metrics/bola/bola.py
@@ -34,9 +34,7 @@ class BOLAMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -123,11 +121,15 @@ class BOLAMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BOLATemplate.generate_evaluation_results(
input=test_case.input,
@@ -156,7 +158,11 @@ class BOLAMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = BOLATemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/competitors/competitors.py b/deepteam/metrics/competitors/competitors.py
index 7a84ee28..f0aa43db 100644
--- a/deepteam/metrics/competitors/competitors.py
+++ b/deepteam/metrics/competitors/competitors.py
@@ -34,9 +34,7 @@ class CompetitorsMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class CompetitorsMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = CompetitorsTemplate.generate_evaluation_results(
input=test_case.input,
@@ -158,7 +160,11 @@ class CompetitorsMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = CompetitorsTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/contracts/contracts.py b/deepteam/metrics/contracts/contracts.py
index 2efb0aa3..8f0ef15e 100644
--- a/deepteam/metrics/contracts/contracts.py
+++ b/deepteam/metrics/contracts/contracts.py
@@ -30,9 +30,7 @@ class ContractsMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -81,11 +79,15 @@ class ContractsMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ContractsTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
@@ -111,7 +113,11 @@ class ContractsMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ContractsTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
diff --git a/deepteam/metrics/debug_access/debug_access.py b/deepteam/metrics/debug_access/debug_access.py
index 60a143a4..0a78dd72 100644
--- a/deepteam/metrics/debug_access/debug_access.py
+++ b/deepteam/metrics/debug_access/debug_access.py
@@ -30,9 +30,7 @@ class DebugAccessMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -81,11 +79,15 @@ class DebugAccessMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = DebugAccessTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
@@ -112,7 +114,11 @@ class DebugAccessMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = DebugAccessTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
diff --git a/deepteam/metrics/excessive_agency/excessive_agency.py b/deepteam/metrics/excessive_agency/excessive_agency.py
index 632a5a75..8abda633 100644
--- a/deepteam/metrics/excessive_agency/excessive_agency.py
+++ b/deepteam/metrics/excessive_agency/excessive_agency.py
@@ -32,9 +32,7 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
self.verbose_mode = verbose_mode
self.purpose = purpose
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -83,11 +81,15 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ExcessiveAgencyTemplate.generate_evaluation_results(
input=test_case.input,
@@ -116,7 +118,11 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ExcessiveAgencyTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/graphic_content/graphic_content.py b/deepteam/metrics/graphic_content/graphic_content.py
index 11955bc2..0714c23b 100644
--- a/deepteam/metrics/graphic_content/graphic_content.py
+++ b/deepteam/metrics/graphic_content/graphic_content.py
@@ -31,9 +31,7 @@ class GraphicMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -82,11 +80,15 @@ class GraphicMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = GraphicTemplate.generate_evaluation_results(
input=test_case.input,
@@ -115,7 +117,11 @@ class GraphicMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = GraphicTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/hallucination/hallucination.py b/deepteam/metrics/hallucination/hallucination.py
index 99a4413f..19145c09 100644
--- a/deepteam/metrics/hallucination/hallucination.py
+++ b/deepteam/metrics/hallucination/hallucination.py
@@ -34,9 +34,7 @@ class HallucinationMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class HallucinationMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HallucinationTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -157,7 +159,11 @@ class HallucinationMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HallucinationTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/harm/harm.py b/deepteam/metrics/harm/harm.py
index 627ec0e2..21cab270 100644
--- a/deepteam/metrics/harm/harm.py
+++ b/deepteam/metrics/harm/harm.py
@@ -31,9 +31,7 @@ class HarmMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -82,11 +80,15 @@ class HarmMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HarmTemplate.generate_evaluation_results(
input=test_case.input,
@@ -115,7 +117,11 @@ class HarmMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HarmTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/hijacking/hijacking.py b/deepteam/metrics/hijacking/hijacking.py
index 80027162..f190711e 100644
--- a/deepteam/metrics/hijacking/hijacking.py
+++ b/deepteam/metrics/hijacking/hijacking.py
@@ -32,9 +32,7 @@ class HijackingMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -123,11 +121,15 @@ class HijackingMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HijackingTemplate.generate_evaluation_results(
input=test_case.input,
@@ -156,7 +158,11 @@ class HijackingMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = HijackingTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/illegal_activity/illegal_activity.py b/deepteam/metrics/illegal_activity/illegal_activity.py
index 93c97b5b..c425c470 100644
--- a/deepteam/metrics/illegal_activity/illegal_activity.py
+++ b/deepteam/metrics/illegal_activity/illegal_activity.py
@@ -31,9 +31,7 @@ class IllegalMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -82,11 +80,15 @@ class IllegalMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = IllegalTemplate.generate_evaluation_results(
input=test_case.input,
@@ -115,7 +117,11 @@ class IllegalMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = IllegalTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/imitation/imitation.py b/deepteam/metrics/imitation/imitation.py
index 4ee7a3e4..86c52df9 100644
--- a/deepteam/metrics/imitation/imitation.py
+++ b/deepteam/metrics/imitation/imitation.py
@@ -34,9 +34,7 @@ class ImitationMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -123,11 +121,15 @@ class ImitationMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ImitationTemplate.generate_evaluation_results(
input=test_case.input,
@@ -156,7 +158,11 @@ class ImitationMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ImitationTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/intellectual_property/intellectual_property.py b/deepteam/metrics/intellectual_property/intellectual_property.py
index 94962180..20099484 100644
--- a/deepteam/metrics/intellectual_property/intellectual_property.py
+++ b/deepteam/metrics/intellectual_property/intellectual_property.py
@@ -32,9 +32,7 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -127,11 +125,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = IntellectualPropertyTemplate.generate_evaluation_results(
actual_output=actual_output,
@@ -159,7 +161,11 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = IntellectualPropertyTemplate.generate_evaluation_results(
actual_output=actual_output,
diff --git a/deepteam/metrics/misinformation/misinformation.py b/deepteam/metrics/misinformation/misinformation.py
index 81c1d854..52098f2a 100644
--- a/deepteam/metrics/misinformation/misinformation.py
+++ b/deepteam/metrics/misinformation/misinformation.py
@@ -31,9 +31,7 @@ class MisinformationMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -82,11 +80,15 @@ class MisinformationMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = MisinformationTemplate.generate_evaluation_results(
input=test_case.input,
@@ -115,7 +117,11 @@ class MisinformationMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = MisinformationTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/overreliance/overreliance.py b/deepteam/metrics/overreliance/overreliance.py
index f0bcf8e9..5f390b2c 100644
--- a/deepteam/metrics/overreliance/overreliance.py
+++ b/deepteam/metrics/overreliance/overreliance.py
@@ -34,9 +34,7 @@ class OverrelianceMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -125,11 +123,15 @@ class OverrelianceMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = OverrelianceTemplate.generate_evaluation_results(
input=test_case.input,
@@ -158,7 +160,11 @@ class OverrelianceMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = OverrelianceTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/personal_safety/personal_safety.py b/deepteam/metrics/personal_safety/personal_safety.py
index c7ae17b2..584c37d8 100644
--- a/deepteam/metrics/personal_safety/personal_safety.py
+++ b/deepteam/metrics/personal_safety/personal_safety.py
@@ -33,9 +33,7 @@ class SafetyMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -84,11 +82,15 @@ class SafetyMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SafetyTemplate.generate_evaluation_results(
input=test_case.input,
@@ -117,7 +119,11 @@ class SafetyMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SafetyTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/pii/pii.py b/deepteam/metrics/pii/pii.py
index f93a7c80..2940b11a 100644
--- a/deepteam/metrics/pii/pii.py
+++ b/deepteam/metrics/pii/pii.py
@@ -34,9 +34,7 @@ class PIIMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -165,11 +163,15 @@ class PIIMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = PIITemplate.generate_evaluation_results(
input=test_case.input,
@@ -199,7 +201,11 @@ class PIIMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = PIITemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/prompt_extraction/prompt_extraction.py b/deepteam/metrics/prompt_extraction/prompt_extraction.py
index b0a5be32..d5c6f305 100644
--- a/deepteam/metrics/prompt_extraction/prompt_extraction.py
+++ b/deepteam/metrics/prompt_extraction/prompt_extraction.py
@@ -32,9 +32,7 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -123,11 +121,15 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = PromptExtractionTemplate.generate_evaluation_results(
input=test_case.input,
@@ -156,7 +158,11 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = PromptExtractionTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/rbac/rbac.py b/deepteam/metrics/rbac/rbac.py
index 26498b05..3915a1ec 100644
--- a/deepteam/metrics/rbac/rbac.py
+++ b/deepteam/metrics/rbac/rbac.py
@@ -34,9 +34,7 @@ class RBACMetric(BaseRedTeamingMetric):
except:
self.system_prompt = ""
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -87,12 +85,16 @@ class RBACMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
-
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
+
prompt = RBACTemplate.generate_evaluation_results(
input=test_case.input,
actual_output=actual_output,
@@ -120,7 +122,11 @@ class RBACMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = RBACTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/shell_injection/shell_injection.py b/deepteam/metrics/shell_injection/shell_injection.py
index d0ba2afc..60d2735d 100644
--- a/deepteam/metrics/shell_injection/shell_injection.py
+++ b/deepteam/metrics/shell_injection/shell_injection.py
@@ -17,6 +17,7 @@ from deepteam.metrics.shell_injection.template import (
)
from deepteam.utils import format_turns
+
class ShellInjectionMetric(BaseRedTeamingMetric):
def __init__(
self,
@@ -29,9 +30,7 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -80,11 +79,15 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ShellInjectionTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
@@ -111,7 +114,11 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ShellInjectionTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
diff --git a/deepteam/metrics/sql_injection/sql_injection.py b/deepteam/metrics/sql_injection/sql_injection.py
index d355af08..6e79c4ca 100644
--- a/deepteam/metrics/sql_injection/sql_injection.py
+++ b/deepteam/metrics/sql_injection/sql_injection.py
@@ -30,9 +30,7 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -79,11 +77,15 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SQLInjectionTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
@@ -110,7 +112,11 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SQLInjectionTemplate.generate_evaluation_results(
input=test_case.input, actual_output=actual_output
diff --git a/deepteam/metrics/ssrf/ssrf.py b/deepteam/metrics/ssrf/ssrf.py
index 9239e8c0..8a0e5fcb 100644
--- a/deepteam/metrics/ssrf/ssrf.py
+++ b/deepteam/metrics/ssrf/ssrf.py
@@ -19,6 +19,7 @@ from deepteam.metrics.ssrf.template import (
)
from deepteam.utils import format_turns
+
class SSRFMetric(BaseRedTeamingMetric):
def __init__(
self,
@@ -33,9 +34,7 @@ class SSRFMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -124,11 +123,15 @@ class SSRFMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SSRFTemplate.generate_evaluation_results(
input=test_case.input,
@@ -157,7 +160,11 @@ class SSRFMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = SSRFTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/metrics/toxicity/toxicity.py b/deepteam/metrics/toxicity/toxicity.py
index 19d5f55c..4bb22a42 100644
--- a/deepteam/metrics/toxicity/toxicity.py
+++ b/deepteam/metrics/toxicity/toxicity.py
@@ -31,9 +31,7 @@ class ToxicityMetric(BaseRedTeamingMetric):
self.async_mode = async_mode
self.verbose_mode = verbose_mode
- def measure(
- self, test_case: RTTestCase
- ) -> float:
+ def measure(self, test_case: RTTestCase) -> float:
self.evaluation_cost = 0 if self.using_native_model else None
with metric_progress_indicator(self, _show_indicator=False):
@@ -82,11 +80,15 @@ class ToxicityMetric(BaseRedTeamingMetric):
async def _a_evaluate(
self, test_case: RTTestCase
) -> Tuple[Union[int, float], str]:
-
+
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ToxicityTemplate.generate_evaluation_results(
input=test_case.input,
@@ -115,7 +117,11 @@ class ToxicityMetric(BaseRedTeamingMetric):
if test_case.turns is not None:
actual_output = format_turns(test_case.turns)
else:
- actual_output = "AI'S ACTUAL OUTPUT TO EVALUATE: \n" + test_case.actual_output + "\n"
+ actual_output = (
+ "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
+ + test_case.actual_output
+ + "\n"
+ )
prompt = ToxicityTemplate.generate_evaluation_results(
input=test_case.input,
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index d9ac816b..7e364c3d 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -106,9 +106,7 @@ class RedTeamer:
and self.test_cases is not None
and len(self.test_cases) > 0
):
- simulated_test_cases: List[RTTestCase] = (
- self.test_cases
- )
+ simulated_test_cases: List[RTTestCase] = self.test_cases
else:
self.attack_simulator.model_callback = model_callback
simulated_test_cases: List[RTTestCase] = (
@@ -210,9 +208,7 @@ class RedTeamer:
and self.test_cases is not None
and len(self.test_cases) > 0
):
- simulated_test_cases: List[RTTestCase] = (
- self.test_cases
- )
+ simulated_test_cases: List[RTTestCase] = self.test_cases
else:
self.attack_simulator.model_callback = model_callback
simulated_test_cases: List[RTTestCase] = (
@@ -417,7 +413,9 @@ class RedTeamer:
simulated_test_case.input, simulated_test_case.turns
)
else:
- actual_output = await model_callback(simulated_test_case.input)
+ actual_output = await model_callback(
+ simulated_test_case.input
+ )
except Exception:
if ignore_errors:
red_teaming_test_case.error = (
diff --git a/deepteam/red_teamer/risk_assessment.py b/deepteam/red_teamer/risk_assessment.py
index c1db9ff6..48473b43 100644
--- a/deepteam/red_teamer/risk_assessment.py
+++ b/deepteam/red_teamer/risk_assessment.py
@@ -93,9 +93,9 @@ class RiskAssessment(BaseModel):
def __init__(self, **data):
super().__init__(**data)
- self.test_cases: TestCasesList = TestCasesList[
- RTTestCase
- ](self.test_cases)
+ self.test_cases: TestCasesList = TestCasesList[RTTestCase](
+ self.test_cases
+ )
def save(self, to: str) -> str:
try:
@@ -134,9 +134,7 @@ def construct_risk_assessment_overview(
VulnerabilityType,
List[RTTestCase],
] = {}
- attack_method_to_cases: Dict[
- str, List[RTTestCase]
- ] = {}
+ attack_method_to_cases: Dict[str, List[RTTestCase]] = {}
errored = 0
for test_case in red_teaming_test_cases:
diff --git a/deepteam/test_case/__init__.py b/deepteam/test_case/__init__.py
index 4822859d..e66dffcb 100644
--- a/deepteam/test_case/__init__.py
+++ b/deepteam/test_case/__init__.py
@@ -1 +1 @@
-from .test_case import RTTestCase, RTTurn
\ No newline at end of file
+from .test_case import RTTestCase, RTTurn
diff --git a/deepteam/test_case/test_case.py b/deepteam/test_case/test_case.py
index b0485939..f54c4e36 100644
--- a/deepteam/test_case/test_case.py
+++ b/deepteam/test_case/test_case.py
@@ -7,6 +7,7 @@ from deepeval.test_case import LLMTestCase, Turn
class RTTurn(Turn):
pass
+
class RTTestCase(LLMTestCase):
vulnerability: str
input: Optional[str] = None
@@ -41,12 +42,14 @@ class RTTestCase(LLMTestCase):
if actual_output is not None:
if not isinstance(actual_output, str):
raise TypeError("'actual_output' must be a string")
-
+
if turns is not None:
- if not isinstance(turns, list) or not all(isinstance(turn, RTTurn) for turn in turns):
+ if not isinstance(turns, list) or not all(
+ isinstance(turn, RTTurn) for turn in turns
+ ):
raise TypeError("'turns' must be a list of 'RTTurn'")
-
- if actual_output is not None and turns is not None:
+
+ if actual_output is not None and turns is not None:
raise ValueError(
"An 'RTTestCase' cannot contain both 'actual_output' and 'turns' at the same time."
)
@@ -54,33 +57,33 @@ class RTTestCase(LLMTestCase):
if vulnerability is not None:
if not isinstance(vulnerability, str):
raise TypeError("'vulnerability' must be a string")
-
+
if vulnerability_type is not None:
if not isinstance(vulnerability, str):
raise TypeError("'vulnerability_type' must be an Enum")
-
+
if metadata is not None:
if not isinstance(metadata, dict):
raise TypeError("'metadata' must be a dictionary")
-
+
if attack_method is not None:
if not isinstance(attack_method, str):
raise TypeError("'attack_method' must be a string")
-
+
if risk_category is not None:
if not isinstance(risk_category, str):
raise TypeError("'risk_category' must be a string")
-
+
if score is not None:
if not isinstance(score, float):
raise TypeError("'score' must be a float")
-
+
if reason is not None:
if not isinstance(reason, str):
raise TypeError("'reason' must be a string")
-
+
if error is not None:
if not isinstance(error, str):
raise TypeError("'error' must be a string")
-
- return data
\ No newline at end of file
+
+ return data
diff --git a/deepteam/utils.py b/deepteam/utils.py
index 38f2d363..bbe15fa4 100644
--- a/deepteam/utils.py
+++ b/deepteam/utils.py
@@ -16,14 +16,15 @@ def validate_model_callback_signature(
"`model_callback` must not be async. `async_mode` has been set to False."
)
+
def format_turns(turns: List[RTTurn]):
if not turns:
raise ValueError("There are no 'turns' to format.")
-
+
formatted_turns = "Full Conversation To Evaluate: \n"
for turn in turns:
formatted_turns += f"Role: {turn.role} \n"
formatted_turns += f"Content: {turn.content} \n\n"
formatted_turns += "End of conversation. \n"
- return formatted_turns
\ No newline at end of file
+ return formatted_turns
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index 7bd7cf62..71a911a1 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -179,7 +179,7 @@ class CustomVulnerability(BaseVulnerability):
purpose: str = None,
attacks_per_vulnerability_type: int = 1,
) -> List[RTTestCase]:
-
+
self.purpose = purpose
self.simulator_model, self.using_native_model = initialize_model(
@@ -197,7 +197,7 @@ class CustomVulnerability(BaseVulnerability):
type,
attacks_per_vulnerability_type,
self.custom_prompt,
- self.purpose
+ self.purpose,
)
)
@@ -237,7 +237,7 @@ class CustomVulnerability(BaseVulnerability):
purpose: str = None,
attacks_per_vulnerability_type: int = 1,
) -> List[RTTestCase]:
-
+
self.purpose = purpose
self.simulator_model, self.using_native_model = initialize_model(
@@ -255,7 +255,7 @@ class CustomVulnerability(BaseVulnerability):
type,
attacks_per_vulnerability_type,
self.custom_prompt,
- self.purpose
+ self.purpose,
)
)
diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
index 22327f39..e95d1faa 100644
--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
@@ -79,9 +79,7 @@ class IntellectualProperty(BaseVulnerability):
purpose, attacks_per_vulnerability_type
)
- results: Dict[IntellectualPropertyType, List[RTTestCase]] = (
- dict()
- )
+ results: Dict[IntellectualPropertyType, List[RTTestCase]] = dict()
for simulated_test_case in simulated_test_cases:
vulnerability_type = simulated_test_case.vulnerability_type
@@ -123,9 +121,7 @@ class IntellectualProperty(BaseVulnerability):
purpose, attacks_per_vulnerability_type
)
- results: Dict[IntellectualPropertyType, List[RTTestCase]] = (
- dict()
- )
+ results: Dict[IntellectualPropertyType, List[RTTestCase]] = dict()
async def process_attack(simulated_test_case: RTTestCase):
vulnerability_type = simulated_test_case.vulnerability_type