DeepTeam is a framework to red team LLMs and AI agents.
removed the metrics map
A-Vamshidd90afdf4054
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex fe57947a..db9fa914 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -172,8 +172,6 @@ class RedTeamer:vulnerabilities=[v.get_name() for v in vulnerabilities],attacks=[a.get_name() for a in attacks],):- # Initialize metric map- metrics_map = self.get_red_teaming_metrics_map(vulnerabilities)# Simulate attacksif (reuse_simulated_attacks@@ -208,9 +206,12 @@ class RedTeamer:)for vulnerability_type, simulated_attacks in pbar:for simulated_attack in simulated_attacks:- metric: BaseRedTeamingMetric = metrics_map.get(- vulnerability_type- )()+ for vulnerability in vulnerabilities:+ if vulnerability_type in vulnerability.types:+ metric: BaseRedTeamingMetric = vulnerability._get_metric(+ vulnerability_type+ )+ breakred_teaming_test_case = RedTeamingTestCase(vulnerability=simulated_attack.vulnerability,@@ -301,9 +302,6 @@ class RedTeamer:vulnerabilities=[v.get_name() for v in vulnerabilities],attacks=[a.get_name() for a in attacks],):- # Initialize metric map- metrics_map = self.get_red_teaming_metrics_map(vulnerabilities)-# Generate attacksif (reuse_simulated_attacks@@ -362,9 +360,9 @@ class RedTeamer:async with semaphore:test_cases = await self._a_evaluate_vulnerability_type(model_callback,+ vulnerabilities,vulnerability_type,attacks,- metrics_map,ignore_errors=ignore_errors,)red_teaming_test_cases.extend(test_cases)@@ -398,7 +396,7 @@ class RedTeamer:simulated_attack: SimulatedAttack,vulnerability: str,vulnerability_type: VulnerabilityType,- metrics_map,+ vulnerabilities: List[BaseVulnerability],ignore_errors: bool,) -> RedTeamingTestCase:red_teaming_test_case = RedTeamingTestCase(@@ -413,7 +411,12 @@ class RedTeamer:red_teaming_test_case.error = simulated_attack.errorreturn red_teaming_test_case- metric: BaseRedTeamingMetric = metrics_map[vulnerability_type]()+ for _vulnerability in vulnerabilities:+ if vulnerability_type in _vulnerability.types:+ metric: BaseRedTeamingMetric = _vulnerability._get_metric(+ vulnerability_type+ )+ breaktry:actual_output = await model_callback(simulated_attack.input)red_teaming_test_case.actual_output = actual_output@@ -445,9 +448,9 @@ class RedTeamer:async def _a_evaluate_vulnerability_type(self,model_callback: CallbackType,+ vulnerabilities: List[BaseVulnerability],vulnerability_type: VulnerabilityType,simulated_attacks: List[SimulatedAttack],- metrics_map,ignore_errors: bool,) -> List[RedTeamingTestCase]:red_teaming_test_cases = await asyncio.gather(@@ -455,9 +458,9 @@ class RedTeamer:self._a_attack(model_callback=model_callback,simulated_attack=simulated_attack,+ vulnerabilities=vulnerabilities,vulnerability=simulated_attack.vulnerability,vulnerability_type=vulnerability_type,- metrics_map=metrics_map,ignore_errors=ignore_errors,)for simulated_attack in simulated_attacks@@ -465,223 +468,6 @@ class RedTeamer:)return red_teaming_test_cases- ##################################################- ### Metrics Map ##################################- ##################################################-- def get_red_teaming_metrics_map(- self, vulnerabilities: List[BaseVulnerability]- ):-- metrics_map = {- #### Bias ####- **{- bias_type: lambda: BiasMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for bias_type in BiasType- },- #### Toxicity ####- **{- toxicity_type: lambda tt=toxicity_type: ToxicityMetric(- model=self.evaluation_model,- toxicity_category=tt.value,- async_mode=self.async_mode,- )- for toxicity_type in ToxicityType- },- #### Misinformation ####- **{- misinformation_type: lambda mt=misinformation_type: MisinformationMetric(- model=self.evaluation_model,- misinformation_category=mt.value,- async_mode=self.async_mode,- )- for misinformation_type in MisinformationType- },- #### Illegal ####- **{- illegal_activity_type: lambda iat=illegal_activity_type: IllegalMetric(- model=self.evaluation_model,- illegal_category=iat.value,- async_mode=self.async_mode,- )- for illegal_activity_type in IllegalActivityType- },- #### Prompt Leakage ####- **{- prompt_leakage_type: lambda: PromptExtractionMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for prompt_leakage_type in PromptLeakageType- },- #### PII Leakage ####- **{- pii_type: lambda: PIIMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for pii_type in PIILeakageType- },- #### Debug Access ####- **{- debug_access_type: lambda: DebugAccessMetric(- model=self.evaluation_model, async_mode=self.async_mode- )- for debug_access_type in DebugAccessType- },- #### Shell Injection ####- **{- shell_injection_type: lambda: ShellInjectionMetric(- model=self.evaluation_model, async_mode=self.async_mode- )- for shell_injection_type in ShellInjectionType- },- #### SQL Injection ####- **{- sql_injection_type: lambda: SQLInjectionMetric(- model=self.evaluation_model, async_mode=self.async_mode- )- for sql_injection_type in SQLInjectionType- },- #### BFLA ####- **{- bfla_type: lambda: BFLAMetric(- purpose=self.target_purpose,- model=self.evaluation_model,- async_mode=self.async_mode,- )- for bfla_type in BFLAType- },- #### BOLA ####- **{- bola_type: lambda: BOLAMetric(- model=self.evaluation_model,- async_mode=self.async_mode,- )- for bola_type in BOLAType- },- #### SSRF ####- **{- ssrf_type: lambda: SSRFMetric(- purpose=self.target_purpose,- model=self.evaluation_model,- async_mode=self.async_mode,- )- for ssrf_type in SSRFType- },- #### RBAC ####- **{- rbac_type: lambda: RBACMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for rbac_type in RBACType- },- #### Excessive Agency ####- **{- excessive_agency_type: lambda: ExcessiveAgencyMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for excessive_agency_type in ExcessiveAgencyType- },- #### Robustness ####- RobustnessType.HIJACKING: lambda: HijackingMetric(- purpose=self.target_purpose,- model=self.evaluation_model,- async_mode=self.async_mode,- ),- RobustnessType.INPUT_OVERRELIANCE: lambda: OverrelianceMetric(- purpose=self.target_purpose,- model=self.evaluation_model,- async_mode=self.async_mode,- ),- #### Intellectual Property ####- **{- ip_type: lambda: IntellectualPropertyMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for ip_type in IntellectualPropertyType- },- #### Competition ####- **{- competiton_type: lambda: CompetitorsMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for competiton_type in CompetitionType- },- #### Graphic Content ####- **{- content_type: lambda ct=content_type: GraphicMetric(- model=self.evaluation_model,- graphic_category=ct.value,- async_mode=self.async_mode,- )- for content_type in GraphicContentType- },- #### Personal Safety ####- **{- safety_type: lambda st=safety_type: SafetyMetric(- model=self.evaluation_model,- safety_category=st.value,- async_mode=self.async_mode,- )- for safety_type in PersonalSafetyType- },- #### Agentic Vulnerabilities ####- **{- subversion_type: lambda: SubversionSuccessMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for subversion_type in RecursiveHijackingType- },- **{- extraction_type: lambda: ExtractionSuccessMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- )- for extraction_type in GoalTheftType- },- }-- # Store custom vulnerability instances for dynamic metric assignment- for vulnerability in vulnerabilities:- if isinstance(vulnerability, CustomVulnerability):- for vuln_type in vulnerability.get_types():- metric = vulnerability.get_metric()- if metric:- metrics_map[vuln_type] = lambda: metric- else:- criteria = vulnerability.get_criteria()- if not criteria:- raise ValueError(- f"CustomVulnerability '{vulnerability.get_name()}' must provide a 'criteria' parameter that defines what should be evaluated."- )-- metrics_map[vuln_type] = lambda hc=criteria: HarmMetric(- model=self.evaluation_model,- harm_category=hc,- async_mode=self.async_mode,- )-- self.metrics_map = metrics_map- return metrics_map-def save_test_cases_as_simulated_attacks(self, test_cases: List[RedTeamingTestCase]):diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.pyindex afced966..d51acb80 100644--- a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py@@ -14,6 +14,7 @@ ExcessiveAgencyLiteral = Literal["functionality", "permissions", "autonomy"]class ExcessiveAgency(BaseVulnerability):def __init__(self,+ purpose: str = None,types: Optional[List[ExcessiveAgencyLiteral]] = [type.value for type in ExcessiveAgencyType],@@ -21,17 +22,18 @@ class ExcessiveAgency(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ExcessiveAgencyType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(- self,- purpose: str = None,+ self,+ type: ExcessiveAgencyType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return ExcessiveAgencyMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py b/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.pyindex e3f3bb28..065348b5 100644--- a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py@@ -13,6 +13,7 @@ GoalTheftLiteralType = Literal[class GoalTheft(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[GoalTheftLiteralType]] = [type.value for type in GoalTheftType],@@ -20,17 +21,18 @@ class GoalTheft(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=GoalTheftType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: GoalTheftType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return ExtractionSuccessMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.pyindex ccb1c39d..dbd76274 100644--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py@@ -17,6 +17,7 @@ RecursiveHijackingLiteralType = Literal[class RecursiveHijacking(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[RecursiveHijackingLiteralType]] = [type.value for type in RecursiveHijackingType],@@ -24,17 +25,18 @@ class RecursiveHijacking(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=RecursiveHijackingType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: RecursiveHijackingType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return SubversionSuccessMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/agentic/robustness/robustness.py b/deepteam/vulnerabilities/agentic/robustness/robustness.pyindex 30f6ce06..b8cd75c5 100644--- a/deepteam/vulnerabilities/agentic/robustness/robustness.py+++ b/deepteam/vulnerabilities/agentic/robustness/robustness.py@@ -11,6 +11,7 @@ RobustnessLiteral = Literal["input overreliance", "hijacking"]class Robustness(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[RobustnessLiteral]] = [type.value for type in RobustnessType],@@ -18,26 +19,27 @@ class Robustness(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=RobustnessType)+ self.purpose = purposesuper().__init__(types=enum_types)+ # TODO: Different metrics for different types. Forces us to use type in the `_get_metric` call.def _get_metric(self,type: RobustnessType,- purpose: str,model: str = None,async_mode: bool = True,verbose_mode: bool = False):if type == RobustnessType.HIJACKING:return HijackingMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_mode)if type == RobustnessType.INPUT_OVERRELIANCE:return OverrelianceMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex 6c539b91..c45b0f91 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -15,22 +15,24 @@ BFLALiteral = Literal[class BFLA(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[BFLALiteral]] = [type.value for type in BFLAType],):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=BFLAType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(- self,- purpose: str,+ self,+ type: BFLAType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return BFLAMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex a6820257..f9507610 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -9,6 +9,7 @@ BiasLiteralType = Literal["religion", "politics", "gender", "race"]class Bias(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[BiasLiteralType]] = [type.value for type in BiasType],@@ -16,17 +17,18 @@ class Bias(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=BiasType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: BiasType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return BiasMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex 990605a2..26570ed7 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -24,6 +24,7 @@ class BOLA(BaseVulnerability):def _get_metric(self,+ type: BOLAType,model: str = None,async_mode: bool = True,verbose_mode: bool = Falsediff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex 5bd767cb..f23eaf44 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -17,6 +17,7 @@ CompetitionLiteralType = Literal[class Competition(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[CompetitionLiteralType]] = [type.value for type in CompetitionType],@@ -24,17 +25,18 @@ class Competition(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=CompetitionType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: CompetitionType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return CompetitorsMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex 5b29d72f..49730a2c 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -36,7 +36,7 @@ class CustomVulnerability(BaseVulnerability):def get_custom_prompt(self) -> Optional[str]:return self.custom_prompt- def get_metric(self) -> Optional[BaseRedTeamingMetric]:+ def _get_metric(self, type: Enum) -> Optional[BaseRedTeamingMetric]:return self.metricdef get_criteria(self) -> str:diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex 193867ae..355300fb 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -26,6 +26,7 @@ class DebugAccess(BaseVulnerability):def _get_metric(self,+ type: DebugAccessType,model: str = None,async_mode: bool = True,verbose_mode: bool = Falsediff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 18b95fdc..24e99037 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -13,6 +13,7 @@ GraphicContentLiteral = Literal[class GraphicContent(BaseVulnerability):def __init__(self,+ graphic_category: str,types: Optional[List[GraphicContentLiteral]] = [type.value for type in GraphicContentType],@@ -20,17 +21,18 @@ class GraphicContent(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=GraphicContentType)+ self.graphic_category = graphic_categorysuper().__init__(types=enum_types)def _get_metric(self,- graphic_category: str,+ type: GraphicContentType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return GraphicMetric(- graphic_category=graphic_category,+ graphic_category=self.graphic_category,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex 0138396b..25c04009 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -19,6 +19,7 @@ IllegalActivityLiteral = Literal[class IllegalActivity(BaseVulnerability):def __init__(self,+ illegal_category: str,types: Optional[List[IllegalActivityLiteral]] = [type.value for type in IllegalActivityType],@@ -26,17 +27,18 @@ class IllegalActivity(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=IllegalActivityType)+ self.illegal_category = illegal_categorysuper().__init__(types=enum_types)def _get_metric(self,- illegal_category: str,+ type: IllegalActivityType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return IllegalMetric(- illegal_category=illegal_category,+ illegal_category=self.illegal_category,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex c7bf2167..6b3fd517 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -18,6 +18,7 @@ IntellectualPropertyLiteral = Literal[class IntellectualProperty(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[IntellectualPropertyLiteral]] = [type.value for type in IntellectualPropertyType],@@ -25,17 +26,18 @@ class IntellectualProperty(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=IntellectualPropertyType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: IntellectualPropertyType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return IntellectualPropertyMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex 15589cb8..8973ca61 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -14,6 +14,7 @@ MisinformationLiteral = Literal[class Misinformation(BaseVulnerability):def __init__(self,+ misinformation_category: str,types: Optional[List[MisinformationLiteral]] = [type.value for type in MisinformationType],@@ -21,17 +22,18 @@ class Misinformation(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=MisinformationType)+ self.misinformation_category = misinformation_categorysuper().__init__(types=enum_types)def _get_metric(self,- misinformation_category: str,+ type: MisinformationType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return MisinformationMetric(- misinformation_category=misinformation_category,+ misinformation_category=self.misinformation_category,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex 5f1a331a..eddf8eb1 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -17,6 +17,7 @@ PersonalSafetyLiteral = Literal[class PersonalSafety(BaseVulnerability):def __init__(self,+ safety_category: str,types: Optional[List[PersonalSafetyLiteral]] = [type.value for type in PersonalSafetyType],@@ -24,17 +25,18 @@ class PersonalSafety(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=PersonalSafetyType)+ self.safety_category = safety_categorysuper().__init__(types=enum_types)def _get_metric(self,- safety_category: str,+ type: PersonalSafetyType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return SafetyMetric(- safety_category=safety_category,+ safety_category=self.safety_category,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex ab5beb34..9c2a2dd5 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -16,6 +16,7 @@ PIILeakageLiteral = Literal[class PIILeakage(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[PIILeakageLiteral]] = [type.value for type in PIILeakageType],@@ -23,17 +24,18 @@ class PIILeakage(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=PIILeakageType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: PIILeakageType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return PIIMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex 10571c07..613f950a 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -16,6 +16,7 @@ PromptLeakageLiteral = Literal[class PromptLeakage(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[PromptLeakageLiteral]] = [type.value for type in PromptLeakageType],@@ -23,17 +24,18 @@ class PromptLeakage(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=PromptLeakageType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: PromptLeakageType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return PromptExtractionMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex 0a4b3918..12d94d4d 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -15,22 +15,24 @@ RBACLiteral = Literal[class RBAC(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[RBACLiteral]] = [type.value for type in RBACType],):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=RBACType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: RBACType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return RBACMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex 25a1e390..5a34247e 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -26,6 +26,7 @@ class ShellInjection(BaseVulnerability):def _get_metric(self,+ type: ShellInjectionType,model: str = None,async_mode: bool = True,verbose_mode: bool = Falsediff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex f88ed5b1..1cb14654 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -26,6 +26,7 @@ class SQLInjection(BaseVulnerability):def _get_metric(self,+ type: SQLInjectionType,model: str = None,async_mode: bool = True,verbose_mode: bool = Falsediff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex 53c3b3b8..2b0cc696 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -15,22 +15,24 @@ SSRFLiteral = Literal[class SSRF(BaseVulnerability):def __init__(self,+ purpose: str,types: Optional[List[SSRFLiteral]] = [type.value for type in SSRFType],):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=SSRFType)+ self.purpose = purposesuper().__init__(types=enum_types)def _get_metric(self,- purpose: str,+ type: SSRFType,model: str = None,async_mode: bool = True,verbose_mode: bool = False):return SSRFMetric(- purpose=purpose,+ purpose=self.purpose,model=model,async_mode=async_mode,verbose_mode=verbose_modediff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex a83c134a..d268a88b 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -11,6 +11,7 @@ ToxicityLiteral = Literal["profanity", "insults", "threats", "mockery"]class Toxicity(BaseVulnerability):def __init__(self,+ toxicity_category: str,types: Optional[List[ToxicityLiteral]] = [type.value for type in ToxicityType],@@ -18,17 +19,18 @@ class Toxicity(BaseVulnerability):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ToxicityType)+ self.toxicity_category = toxicity_categorysuper().__init__(types=enum_types)def _get_metric(self,- toxicity_category: str,+model: str = None,async_mode: bool = True,verbose_mode: bool = False):return ToxicityMetric(- toxicity_category=toxicity_category,+ toxicity_category=self.toxicity_category,model=model,async_mode=async_mode,verbose_mode=verbose_mode