DeepTeam is a framework to red team LLMs and AI agents.
Updated tests for vulnerabilities
A-Vamshi14ef77ced8a6
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex 49730a2c..91d3d5d7 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -1,8 +1,10 @@-from typing import List, Optional+from typing import List, Optional, Unionfrom enum import Enum+from deepeval.models import DeepEvalBaseLLM+from deepteam.vulnerabilities import BaseVulnerability-from deepteam.metrics import BaseRedTeamingMetric+from deepteam.metrics import BaseRedTeamingMetric, HarmMetricclass CustomVulnerability(BaseVulnerability):@@ -17,6 +19,9 @@ class CustomVulnerability(BaseVulnerability):types: Optional[List[str]] = None,custom_prompt: Optional[str] = None,metric: Optional[BaseRedTeamingMetric] = None,+ model: Optional[Union[str, DeepEvalBaseLLM]] = None,+ async_mode: bool = True,+ verbose_mode: bool = False,):self.name = name@@ -26,8 +31,19 @@ class CustomVulnerability(BaseVulnerability):)self.custom_prompt = custom_prompt- self.metric = metricself.criteria = criteria.strip()+ self.model = model+ self.async_mode = async_mode+ self.verbose_mode = verbose_mode+ if metric:+ self.metric = metric+ else:+ self.metric = HarmMetric(+ harm_category=self.criteria,+ model=self.model,+ async_mode=self.async_mode,+ verbose_mode=self.verbose_mode+ )super().__init__(self.types)def get_name(self) -> str:diff --git a/tests/test_core/test_red_teaming.py b/tests/test_core/test_red_teaming.pyindex 7610558f..c8f9e1cb 100644--- a/tests/test_core/test_red_teaming.py+++ b/tests/test_core/test_red_teaming.py@@ -9,6 +9,13 @@ async def model_callback(input: str) -> str:def test_red_teaming():+ PURPOSE="purpose"+ TOXICITY="category"+ GRAPHIC="category"+ SAFETY="category"+ MISINFORMATION="category"+ ILLEGAL="category"+custom_vulnerability = CustomVulnerability(criteria="Criteria",name="Name",@@ -17,28 +24,28 @@ def test_red_teaming():)vulnerabilities = [- Bias(),- Toxicity(),- Misinformation(),- IllegalActivity(),- PromptLeakage(),- PIILeakage(),- BFLA(),+ Bias(purpose=PURPOSE),+ Toxicity(toxicity_category=TOXICITY),+ Misinformation(misinformation_category=MISINFORMATION),+ IllegalActivity(illegal_category=ILLEGAL),+ PromptLeakage(purpose=PURPOSE),+ PIILeakage(purpose=PURPOSE),+ BFLA(purpose=PURPOSE),BOLA(),- RBAC(),+ RBAC(purpose=PURPOSE),DebugAccess(),ShellInjection(),SQLInjection(),- SSRF(),- IntellectualProperty(),- Competition(),- GraphicContent(),- PersonalSafety(),+ SSRF(purpose=PURPOSE),+ IntellectualProperty(purpose=PURPOSE),+ Competition(purpose=PURPOSE),+ GraphicContent(graphic_category=GRAPHIC),+ PersonalSafety(safety_category=SAFETY),# Agentic vulnerabilities- GoalTheft(),- RecursiveHijacking(),- Robustness(),- ExcessiveAgency(),+ GoalTheft(purpose=PURPOSE),+ RecursiveHijacking(purpose=PURPOSE),+ Robustness(purpose=PURPOSE),+ ExcessiveAgency(purpose=PURPOSE),custom_vulnerability,]diff --git a/tests/test_core/test_vulnerabilities/test_all.py b/tests/test_core/test_vulnerabilities/test_all.pyindex 6ea921e4..b0d3b0d8 100644--- a/tests/test_core/test_vulnerabilities/test_all.py+++ b/tests/test_core/test_vulnerabilities/test_all.py@@ -8,6 +8,13 @@ async def model_callback(input: str) -> str:def test_red_teaming():+ PURPOSE="purpose"+ TOXICITY="category"+ GRAPHIC="category"+ SAFETY="category"+ MISINFORMATION="category"+ ILLEGAL="category"+custom_vulnerability = CustomVulnerability(criteria="Criteria",name="Name",@@ -16,28 +23,28 @@ def test_red_teaming():)vulnerabilities = [- Bias(),- Toxicity(),- Misinformation(),- IllegalActivity(),- PromptLeakage(),- PIILeakage(),- BFLA(),+ Bias(purpose=PURPOSE),+ Toxicity(toxicity_category=TOXICITY),+ Misinformation(misinformation_category=MISINFORMATION),+ IllegalActivity(illegal_category=ILLEGAL),+ PromptLeakage(purpose=PURPOSE),+ PIILeakage(purpose=PURPOSE),+ BFLA(purpose=PURPOSE),BOLA(),- RBAC(),+ RBAC(purpose=PURPOSE),DebugAccess(),ShellInjection(),SQLInjection(),- SSRF(),- IntellectualProperty(),- Competition(),- GraphicContent(),- PersonalSafety(),+ SSRF(purpose=PURPOSE),+ IntellectualProperty(purpose=PURPOSE),+ Competition(purpose=PURPOSE),+ GraphicContent(graphic_category=GRAPHIC),+ PersonalSafety(safety_category=SAFETY),# Agentic vulnerabilities- GoalTheft(),- RecursiveHijacking(),- Robustness(),- ExcessiveAgency(),+ GoalTheft(purpose=PURPOSE),+ RecursiveHijacking(purpose=PURPOSE),+ Robustness(purpose=PURPOSE),+ ExcessiveAgency(purpose=PURPOSE),custom_vulnerability,]diff --git a/tests/test_core/test_vulnerabilities/test_bfla.py b/tests/test_core/test_vulnerabilities/test_bfla.pyindex c889f46d..290ee727 100644--- a/tests/test_core/test_vulnerabilities/test_bfla.py+++ b/tests/test_core/test_vulnerabilities/test_bfla.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.bfla import BFLATypeclass TestBFLA:+ PURPOSE="purpose"def test_bfla_all_types(self):types = [@@ -12,28 +13,28 @@ class TestBFLA:"function_bypass","authorization_bypass",]- bfla = BFLA(types=types)+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)assert sorted(type.value for type in bfla.types) == sorted(types)def test_bfla_all_types_default(self):- bfla = BFLA()+ bfla = BFLA(purpose=TestBFLA.PURPOSE, )assert sorted(type.value for type in bfla.types) == sorted(type.value for type in BFLAType)def test_bfla_privilege_escalation(self):types = ["privilege_escalation"]- bfla = BFLA(types=types)+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)assert sorted(type.value for type in bfla.types) == sorted(types)def test_bfla_function_bypass(self):types = ["function_bypass"]- bfla = BFLA(types=types)+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)assert sorted(type.value for type in bfla.types) == sorted(types)def test_bfla_authorization_bypass(self):types = ["authorization_bypass"]- bfla = BFLA(types=types)+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)assert sorted(type.value for type in bfla.types) == sorted(types)def test_bfla_all_types_invalid(self):@@ -44,4 +45,4 @@ class TestBFLA:"invalid",]with pytest.raises(ValueError):- BFLA(types=types)+ BFLA(purpose=TestBFLA.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_bias.py b/tests/test_core/test_vulnerabilities/test_bias.pyindex fe792d01..ac4a678a 100644--- a/tests/test_core/test_vulnerabilities/test_bias.py+++ b/tests/test_core/test_vulnerabilities/test_bias.py@@ -5,39 +5,40 @@ from deepteam.vulnerabilities.bias import BiasTypeclass TestBias:+ PURPOSE="purpose"def test_bias_all_types(self):types = ["gender", "politics", "religion", "race"]- bias = Bias(types=types)+ bias = Bias(purpose=TestBias.PURPOSE, types=types)assert sorted(type.value for type in bias.types) == sorted(types)def test_bias_all_types_default(self):- bias = Bias()+ bias = Bias(purpose=TestBias.PURPOSE, )assert sorted(type.value for type in bias.types) == sorted(type.value for type in BiasType)def test_bias_gender(self):types = ["gender"]- bias = Bias(types=types)+ bias = Bias(purpose=TestBias.PURPOSE, types=types)assert sorted(type.value for type in bias.types) == sorted(types)def test_bias_politics(self):types = ["politics"]- bias = Bias(types=types)+ bias = Bias(purpose=TestBias.PURPOSE, types=types)assert sorted(type.value for type in bias.types) == sorted(types)def test_bias_religion(self):types = ["religion"]- bias = Bias(types=types)+ bias = Bias(purpose=TestBias.PURPOSE, types=types)assert sorted(type.value for type in bias.types) == sorted(types)def test_bias_race(self):types = ["race"]- bias = Bias(types=types)+ bias = Bias(purpose=TestBias.PURPOSE, types=types)assert sorted(type.value for type in bias.types) == sorted(types)def test_bias_all_types_invalid(self):types = ["gender", "politics", "religion", "race", "invalid"]with pytest.raises(ValueError):- Bias(types=types)+ Bias(purpose=TestBias.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_competition.py b/tests/test_core/test_vulnerabilities/test_competition.pyindex 741b2529..4eb21182 100644--- a/tests/test_core/test_vulnerabilities/test_competition.py+++ b/tests/test_core/test_vulnerabilities/test_competition.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.competition import CompetitionTypeclass TestCompetition:+ PURPOSE="purpose"def test_competition_all_types(self):types = [@@ -13,33 +14,33 @@ class TestCompetition:"discreditation","confidential strategies",]- competition = Competition(types=types)+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)assert sorted(type.value for type in competition.types) == sorted(types)def test_competition_all_types_default(self):- competition = Competition()+ competition = Competition(purpose=TestCompetition.PURPOSE, )assert sorted(type.value for type in competition.types) == sorted(type.value for type in CompetitionType)def test_competition_competitor_mention(self):types = ["competitor mention"]- competition = Competition(types=types)+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)assert sorted(type.value for type in competition.types) == sorted(types)def test_competition_market_manipulation(self):types = ["market manipulation"]- competition = Competition(types=types)+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)assert sorted(type.value for type in competition.types) == sorted(types)def test_competition_discreditation(self):types = ["discreditation"]- competition = Competition(types=types)+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)assert sorted(type.value for type in competition.types) == sorted(types)def test_competition_confidential_strategies(self):types = ["confidential strategies"]- competition = Competition(types=types)+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)assert sorted(type.value for type in competition.types) == sorted(types)def test_competition_all_types_invalid(self):@@ -51,4 +52,4 @@ class TestCompetition:"invalid",]with pytest.raises(ValueError):- Competition(types=types)+ Competition(purpose=TestCompetition.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py b/tests/test_core/test_vulnerabilities/test_custom_vulnerability.pyindex d78447d6..626e2dd8 100644--- a/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py+++ b/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py@@ -15,6 +15,7 @@ class TestCustomVulnerability:assert custom_vulnerability.criteria == "Criteria"assert custom_vulnerability.name == "Name"assert custom_vulnerability.custom_prompt == "Custom prompt"+ assert custom_vulnerability.metric is not Noneassert sorted(type.value for type in custom_vulnerability.types) == sorted(["type1", "type2"])diff --git a/tests/test_core/test_vulnerabilities/test_excessive_agency.py b/tests/test_core/test_vulnerabilities/test_excessive_agency.pyindex 1131af0f..50243d65 100644--- a/tests/test_core/test_vulnerabilities/test_excessive_agency.py+++ b/tests/test_core/test_vulnerabilities/test_excessive_agency.py@@ -7,37 +7,38 @@ from deepteam.vulnerabilities.agentic.excessive_agency import (class TestExcessiveAgency:+ PURPOSE="purpose"def test_excessive_agency_all_types(self):types = ["functionality", "permissions", "autonomy"]- excessive_agency = ExcessiveAgency(types=types)+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)assert sorted(type.value for type in excessive_agency.types) == sorted(types)def test_excessive_agency_all_types_default(self):- excessive_agency = ExcessiveAgency()+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, )assert sorted(type.value for type in excessive_agency.types) == sorted(type.value for type in ExcessiveAgencyType)def test_excessive_agency_functionality(self):types = ["functionality"]- excessive_agency = ExcessiveAgency(types=types)+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)assert sorted(type.value for type in excessive_agency.types) == sorted(types)def test_excessive_agency_permissions(self):types = ["permissions"]- excessive_agency = ExcessiveAgency(types=types)+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)assert sorted(type.value for type in excessive_agency.types) == sorted(types)def test_excessive_agency_autonomy(self):types = ["autonomy"]- excessive_agency = ExcessiveAgency(types=types)+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)assert sorted(type.value for type in excessive_agency.types) == sorted(types)@@ -45,4 +46,4 @@ class TestExcessiveAgency:def test_excessive_agency_all_types_invalid(self):types = ["functionality", "permissions", "autonomy", "invalid"]with pytest.raises(ValueError):- ExcessiveAgency(types=types)+ ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_goal_theft.py b/tests/test_core/test_vulnerabilities/test_goal_theft.pyindex 0b4d09b4..2e0af11b 100644--- a/tests/test_core/test_vulnerabilities/test_goal_theft.py+++ b/tests/test_core/test_vulnerabilities/test_goal_theft.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.agentic.goal_theft import GoalTheftTypeclass TestGoalTheft:+ PURPOSE="purpose"def test_goal_theft_all_types(self):types = [@@ -12,28 +13,28 @@ class TestGoalTheft:"cooperative_dialogue","social_engineering",]- goal_theft = GoalTheft(types=types)+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)assert sorted(type.value for type in goal_theft.types) == sorted(types)def test_goal_theft_all_types_default(self):- goal_theft = GoalTheft()+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, )assert sorted(type.value for type in goal_theft.types) == sorted(type.value for type in GoalTheftType)def test_goal_theft_escalating_probing(self):types = ["escalating_probing"]- goal_theft = GoalTheft(types=types)+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)assert sorted(type.value for type in goal_theft.types) == sorted(types)def test_goal_theft_cooperative_dialogue(self):types = ["cooperative_dialogue"]- goal_theft = GoalTheft(types=types)+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)assert sorted(type.value for type in goal_theft.types) == sorted(types)def test_goal_theft_social_engineering(self):types = ["social_engineering"]- goal_theft = GoalTheft(types=types)+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)assert sorted(type.value for type in goal_theft.types) == sorted(types)def test_goal_theft_all_types_invalid(self):@@ -44,4 +45,4 @@ class TestGoalTheft:"invalid",]with pytest.raises(ValueError):- GoalTheft(types=types)+ GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_graphic_content.py b/tests/test_core/test_vulnerabilities/test_graphic_content.pyindex 16250ed1..db17aa53 100644--- a/tests/test_core/test_vulnerabilities/test_graphic_content.py+++ b/tests/test_core/test_vulnerabilities/test_graphic_content.py@@ -5,37 +5,38 @@ from deepteam.vulnerabilities.graphic_content import GraphicContentTypeclass TestGraphicContent:+ GRAPHIC="category"def test_graphic_content_all_types(self):types = ["sexual content", "graphic content", "pornographic content"]- graphic_content = GraphicContent(types=types)+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)assert sorted(type.value for type in graphic_content.types) == sorted(types)def test_graphic_content_all_types_default(self):- graphic_content = GraphicContent()+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, )assert sorted(type.value for type in graphic_content.types) == sorted(type.value for type in GraphicContentType)def test_graphic_content_sexual_content(self):types = ["sexual content"]- graphic_content = GraphicContent(types=types)+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)assert sorted(type.value for type in graphic_content.types) == sorted(types)def test_graphic_content_graphic_content(self):types = ["graphic content"]- graphic_content = GraphicContent(types=types)+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)assert sorted(type.value for type in graphic_content.types) == sorted(types)def test_graphic_content_pornographic_content(self):types = ["pornographic content"]- graphic_content = GraphicContent(types=types)+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)assert sorted(type.value for type in graphic_content.types) == sorted(types)@@ -48,4 +49,4 @@ class TestGraphicContent:"invalid",]with pytest.raises(ValueError):- GraphicContent(types=types)+ GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_illegal_activity.py b/tests/test_core/test_vulnerabilities/test_illegal_activity.pyindex 1ce14eb1..5c78815e 100644--- a/tests/test_core/test_vulnerabilities/test_illegal_activity.py+++ b/tests/test_core/test_vulnerabilities/test_illegal_activity.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.illegal_activity import IllegalActivityTypeclass TestIllegalActivity:+ ILLEGAL="category"def test_illegal_activity_all_types(self):types = [@@ -16,62 +17,62 @@ class TestIllegalActivity:"cybercrime","child exploitation",]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_all_types_default(self):- illegal_activity = IllegalActivity()+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, )assert sorted(type.value for type in illegal_activity.types) == sorted(type.value for type in IllegalActivityType)def test_illegal_activity_weapons(self):types = ["weapons"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_illegal_drugs(self):types = ["illegal drugs"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_violent_crimes(self):types = ["violent crimes"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_non_violent_crimes(self):types = ["non violent crimes"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_sex_crimes(self):types = ["sex crimes"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_cybercrime(self):types = ["cybercrime"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)def test_illegal_activity_child_exploitation(self):types = ["child exploitation"]- illegal_activity = IllegalActivity(types=types)+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)assert sorted(type.value for type in illegal_activity.types) == sorted(types)@@ -88,4 +89,4 @@ class TestIllegalActivity:"invalid",]with pytest.raises(ValueError):- IllegalActivity(types=types)+ IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_intellectual_property.py b/tests/test_core/test_vulnerabilities/test_intellectual_property.pyindex 2c259817..502029f3 100644--- a/tests/test_core/test_vulnerabilities/test_intellectual_property.py+++ b/tests/test_core/test_vulnerabilities/test_intellectual_property.py@@ -7,6 +7,7 @@ from deepteam.vulnerabilities.intellectual_property import (class TestIntellectualProperty:+ PURPOSE="purpose"def test_intellectual_property_all_types(self):types = [@@ -15,41 +16,41 @@ class TestIntellectualProperty:"trademark infringement","patent disclosure",]- intellectual_property = IntellectualProperty(types=types)+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)assert sorted(type.value for type in intellectual_property.types) == sorted(types)def test_intellectual_property_all_types_default(self):- intellectual_property = IntellectualProperty()+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, )assert sorted(type.value for type in intellectual_property.types) == sorted(type.value for type in IntellectualPropertyType)def test_intellectual_property_imitation(self):types = ["imitation"]- intellectual_property = IntellectualProperty(types=types)+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)assert sorted(type.value for type in intellectual_property.types) == sorted(types)def test_intellectual_property_copyright_violations(self):types = ["copyright violations"]- intellectual_property = IntellectualProperty(types=types)+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)assert sorted(type.value for type in intellectual_property.types) == sorted(types)def test_intellectual_property_trademark_infringement(self):types = ["trademark infringement"]- intellectual_property = IntellectualProperty(types=types)+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)assert sorted(type.value for type in intellectual_property.types) == sorted(types)def test_intellectual_property_patent_disclosure(self):types = ["patent disclosure"]- intellectual_property = IntellectualProperty(types=types)+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)assert sorted(type.value for type in intellectual_property.types) == sorted(types)@@ -63,4 +64,4 @@ class TestIntellectualProperty:"invalid",]with pytest.raises(ValueError):- IntellectualProperty(types=types)+ IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_misinformation.py b/tests/test_core/test_vulnerabilities/test_misinformation.pyindex 66225b04..523c92fb 100644--- a/tests/test_core/test_vulnerabilities/test_misinformation.py+++ b/tests/test_core/test_vulnerabilities/test_misinformation.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.misinformation import MisinformationTypeclass TestMisinformation:+ MISINFORMATION="category"def test_misinformation_all_types(self):types = [@@ -12,34 +13,34 @@ class TestMisinformation:"unsupported claims","expertize misrepresentation",]- misinformation = Misinformation(types=types)+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)assert sorted(type.value for type in misinformation.types) == sorted(types)def test_misinformation_all_types_default(self):- misinformation = Misinformation()+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, )assert sorted(type.value for type in misinformation.types) == sorted(type.value for type in MisinformationType)def test_misinformation_factual_errors(self):types = ["factual errors"]- misinformation = Misinformation(types=types)+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)assert sorted(type.value for type in misinformation.types) == sorted(types)def test_misinformation_unsupported_claims(self):types = ["unsupported claims"]- misinformation = Misinformation(types=types)+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)assert sorted(type.value for type in misinformation.types) == sorted(types)def test_misinformation_expertize_misrepresentation(self):types = ["expertize misrepresentation"]- misinformation = Misinformation(types=types)+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)assert sorted(type.value for type in misinformation.types) == sorted(types)@@ -52,4 +53,4 @@ class TestMisinformation:"invalid",]with pytest.raises(ValueError):- Misinformation(types=types)+ Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_personal_safety.py b/tests/test_core/test_vulnerabilities/test_personal_safety.pyindex 6cec5652..b2dbcbcf 100644--- a/tests/test_core/test_vulnerabilities/test_personal_safety.py+++ b/tests/test_core/test_vulnerabilities/test_personal_safety.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.personal_safety import PersonalSafetyTypeclass TestPersonalSafety:+ SAFETY="category"def test_personal_safety_all_types(self):types = [@@ -14,48 +15,48 @@ class TestPersonalSafety:"dangerous challenges","stalking",]- personal_safety = PersonalSafety(types=types)+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)assert sorted(type.value for type in personal_safety.types) == sorted(types)def test_personal_safety_all_types_default(self):- personal_safety = PersonalSafety()+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, )assert sorted(type.value for type in personal_safety.types) == sorted(type.value for type in PersonalSafetyType)def test_personal_safety_bullying(self):types = ["bullying"]- personal_safety = PersonalSafety(types=types)+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)assert sorted(type.value for type in personal_safety.types) == sorted(types)def test_personal_safety_self_harm(self):types = ["self-harm"]- personal_safety = PersonalSafety(types=types)+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)assert sorted(type.value for type in personal_safety.types) == sorted(types)def test_personal_safety_unsafe_practices(self):types = ["unsafe practices"]- personal_safety = PersonalSafety(types=types)+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)assert sorted(type.value for type in personal_safety.types) == sorted(types)def test_personal_safety_dangerous_challenges(self):types = ["dangerous challenges"]- personal_safety = PersonalSafety(types=types)+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)assert sorted(type.value for type in personal_safety.types) == sorted(types)def test_personal_safety_stalking(self):types = ["stalking"]- personal_safety = PersonalSafety(types=types)+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)assert sorted(type.value for type in personal_safety.types) == sorted(types)@@ -70,4 +71,4 @@ class TestPersonalSafety:"invalid",]with pytest.raises(ValueError):- PersonalSafety(types=types)+ PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_pii_leakage.py b/tests/test_core/test_vulnerabilities/test_pii_leakage.pyindex dc1b6c49..c160afea 100644--- a/tests/test_core/test_vulnerabilities/test_pii_leakage.py+++ b/tests/test_core/test_vulnerabilities/test_pii_leakage.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.pii_leakage import PIILeakageTypeclass TestPIILeakage:+ PURPOSE="purpose"def test_pii_leakage_all_types(self):types = [@@ -13,33 +14,33 @@ class TestPIILeakage:"session leak","social manipulation",]- pii_leakage = PIILeakage(types=types)+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)assert sorted(type.value for type in pii_leakage.types) == sorted(types)def test_pii_leakage_all_types_default(self):- pii_leakage = PIILeakage()+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, )assert sorted(type.value for type in pii_leakage.types) == sorted(type.value for type in PIILeakageType)def test_pii_leakage_database_access(self):types = ["api and database access"]- pii_leakage = PIILeakage(types=types)+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)assert sorted(type.value for type in pii_leakage.types) == sorted(types)def test_pii_leakage_direct_disclosure(self):types = ["direct disclosure"]- pii_leakage = PIILeakage(types=types)+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)assert sorted(type.value for type in pii_leakage.types) == sorted(types)def test_pii_leakage_session_leak(self):types = ["session leak"]- pii_leakage = PIILeakage(types=types)+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)assert sorted(type.value for type in pii_leakage.types) == sorted(types)def test_pii_leakage_social_manipulation(self):types = ["social manipulation"]- pii_leakage = PIILeakage(types=types)+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)assert sorted(type.value for type in pii_leakage.types) == sorted(types)def test_pii_leakage_all_types_invalid(self):@@ -51,4 +52,4 @@ class TestPIILeakage:"invalid",]with pytest.raises(ValueError):- PIILeakage(types=types)+ PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_prompt_leakage.py b/tests/test_core/test_vulnerabilities/test_prompt_leakage.pyindex 0ee7ab5d..bdbec163 100644--- a/tests/test_core/test_vulnerabilities/test_prompt_leakage.py+++ b/tests/test_core/test_vulnerabilities/test_prompt_leakage.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.prompt_leakage import PromptLeakageTypeclass TestPromptLeakage:+ PURPOSE="purpose"def test_prompt_leakage_all_types(self):types = [@@ -13,41 +14,41 @@ class TestPromptLeakage:"guard exposure","permissions and roles",]- prompt_leakage = PromptLeakage(types=types)+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)assert sorted(type.value for type in prompt_leakage.types) == sorted(types)def test_prompt_leakage_all_types_default(self):- prompt_leakage = PromptLeakage()+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, )assert sorted(type.value for type in prompt_leakage.types) == sorted(type.value for type in PromptLeakageType)def test_prompt_leakage_secrets_and_credentials(self):types = ["secrets and credentials"]- prompt_leakage = PromptLeakage(types=types)+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)assert sorted(type.value for type in prompt_leakage.types) == sorted(types)def test_prompt_leakage_instructions(self):types = ["instructions"]- prompt_leakage = PromptLeakage(types=types)+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)assert sorted(type.value for type in prompt_leakage.types) == sorted(types)def test_prompt_leakage_guard_exposure(self):types = ["guard exposure"]- prompt_leakage = PromptLeakage(types=types)+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)assert sorted(type.value for type in prompt_leakage.types) == sorted(types)def test_prompt_leakage_permissions_and_roles(self):types = ["permissions and roles"]- prompt_leakage = PromptLeakage(types=types)+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)assert sorted(type.value for type in prompt_leakage.types) == sorted(types)@@ -61,4 +62,4 @@ class TestPromptLeakage:"invalid",]with pytest.raises(ValueError):- PromptLeakage(types=types)+ PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_rbac.py b/tests/test_core/test_vulnerabilities/test_rbac.pyindex 7e51a87e..a4919393 100644--- a/tests/test_core/test_vulnerabilities/test_rbac.py+++ b/tests/test_core/test_vulnerabilities/test_rbac.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.rbac import RBACTypeclass TestRBAC:+ PURPOSE="purpose"def test_rbac_all_types(self):types = [@@ -12,28 +13,28 @@ class TestRBAC:"privilege escalation","unauthorized role assumption",]- rbac = RBAC(types=types)+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)assert sorted(type.value for type in rbac.types) == sorted(types)def test_rbac_all_types_default(self):- rbac = RBAC()+ rbac = RBAC(purpose=TestRBAC.PURPOSE, )assert sorted(type.value for type in rbac.types) == sorted(type.value for type in RBACType)def test_rbac_role_bypass(self):types = ["role bypass"]- rbac = RBAC(types=types)+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)assert sorted(type.value for type in rbac.types) == sorted(types)def test_rbac_privilege_escalation(self):types = ["privilege escalation"]- rbac = RBAC(types=types)+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)assert sorted(type.value for type in rbac.types) == sorted(types)def test_rbac_unauthorized_role_assumption(self):types = ["unauthorized role assumption"]- rbac = RBAC(types=types)+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)assert sorted(type.value for type in rbac.types) == sorted(types)def test_rbac_all_types_invalid(self):@@ -44,4 +45,4 @@ class TestRBAC:"invalid",]with pytest.raises(ValueError):- RBAC(types=types)+ RBAC(purpose=TestRBAC.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py b/tests/test_core/test_vulnerabilities/test_recursive_hijacking.pyindex a29017be..91752a9a 100644--- a/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py+++ b/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py@@ -7,6 +7,7 @@ from deepteam.vulnerabilities.agentic.recursive_hijacking import (class TestRecursiveHijacking:+ PURPOSE="purpose"def test_recursive_hijacking_all_types(self):types = [@@ -14,34 +15,34 @@ class TestRecursiveHijacking:"recursive_objective_chaining","goal_propagation_attacks",]- recursive_hijacking = RecursiveHijacking(types=types)+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)assert sorted(type.value for type in recursive_hijacking.types) == sorted(types)def test_recursive_hijacking_all_types_default(self):- recursive_hijacking = RecursiveHijacking()+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, )assert sorted(type.value for type in recursive_hijacking.types) == sorted(type.value for type in RecursiveHijackingType)def test_recursive_hijacking_self_modifying_goals(self):types = ["self_modifying_goals"]- recursive_hijacking = RecursiveHijacking(types=types)+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)assert sorted(type.value for type in recursive_hijacking.types) == sorted(types)def test_recursive_hijacking_recursive_objective_chaining(self):types = ["recursive_objective_chaining"]- recursive_hijacking = RecursiveHijacking(types=types)+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)assert sorted(type.value for type in recursive_hijacking.types) == sorted(types)def test_recursive_hijacking_goal_propagation_attacks(self):types = ["goal_propagation_attacks"]- recursive_hijacking = RecursiveHijacking(types=types)+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)assert sorted(type.value for type in recursive_hijacking.types) == sorted(types)@@ -54,4 +55,4 @@ class TestRecursiveHijacking:"invalid",]with pytest.raises(ValueError):- RecursiveHijacking(types=types)+ RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_robustness.py b/tests/test_core/test_vulnerabilities/test_robustness.pyindex 70bf827d..26378382 100644--- a/tests/test_core/test_vulnerabilities/test_robustness.py+++ b/tests/test_core/test_vulnerabilities/test_robustness.py@@ -5,29 +5,30 @@ from deepteam.vulnerabilities.agentic.robustness import RobustnessTypeclass TestRobustness:+ PURPOSE="purpose"def test_robustness_all_types(self):types = ["input overreliance", "hijacking"]- robustness = Robustness(types=types)+ robustness = Robustness(purpose=TestRobustness.PURPOSE, types=types)assert sorted(type.value for type in robustness.types) == sorted(types)def test_robustness_all_types_default(self):- robustness = Robustness()+ robustness = Robustness(purpose=TestRobustness.PURPOSE, )assert sorted(type.value for type in robustness.types) == sorted(type.value for type in RobustnessType)def test_robustness_input_overreliance(self):types = ["input overreliance"]- robustness = Robustness(types=types)+ robustness = Robustness(purpose=TestRobustness.PURPOSE, types=types)assert sorted(type.value for type in robustness.types) == sorted(types)def test_robustness_hijacking(self):types = ["hijacking"]- robustness = Robustness(types=types)+ robustness = Robustness(purpose=TestRobustness.PURPOSE, types=types)assert sorted(type.value for type in robustness.types) == sorted(types)def test_robustness_all_types_invalid(self):types = ["input overreliance", "hijacking", "invalid"]with pytest.raises(ValueError):- Robustness(types=types)+ Robustness(purpose=TestRobustness.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_ssrf.py b/tests/test_core/test_vulnerabilities/test_ssrf.pyindex d655fb8c..f810a262 100644--- a/tests/test_core/test_vulnerabilities/test_ssrf.py+++ b/tests/test_core/test_vulnerabilities/test_ssrf.py@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.ssrf import SSRFTypeclass TestSSRF:+ PURPOSE="purpose"def test_ssrf_all_types(self):types = [@@ -12,28 +13,28 @@ class TestSSRF:"cloud_metadata_access","port_scanning",]- ssrf = SSRF(types=types)+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)assert sorted(type.value for type in ssrf.types) == sorted(types)def test_ssrf_all_types_default(self):- ssrf = SSRF()+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, )assert sorted(type.value for type in ssrf.types) == sorted(type.value for type in SSRFType)def test_ssrf_internal_service_access(self):types = ["internal_service_access"]- ssrf = SSRF(types=types)+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)assert sorted(type.value for type in ssrf.types) == sorted(types)def test_ssrf_cloud_metadata_access(self):types = ["cloud_metadata_access"]- ssrf = SSRF(types=types)+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)assert sorted(type.value for type in ssrf.types) == sorted(types)def test_ssrf_port_scanning(self):types = ["port_scanning"]- ssrf = SSRF(types=types)+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)assert sorted(type.value for type in ssrf.types) == sorted(types)def test_ssrf_all_types_invalid(self):@@ -44,4 +45,4 @@ class TestSSRF:"invalid",]with pytest.raises(ValueError):- SSRF(types=types)+ SSRF(purpose=TestSSRF.PURPOSE, types=types)diff --git a/tests/test_core/test_vulnerabilities/test_toxicity.py b/tests/test_core/test_vulnerabilities/test_toxicity.pyindex 5fa9d482..874afb5e 100644--- a/tests/test_core/test_vulnerabilities/test_toxicity.py+++ b/tests/test_core/test_vulnerabilities/test_toxicity.py@@ -5,39 +5,40 @@ from deepteam.vulnerabilities.toxicity import ToxicityTypeclass TestToxicity:+ TOXICITY="category"def test_toxicity_all_types(self):types = ["profanity", "insults", "threats", "mockery"]- toxicity = Toxicity(types=types)+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)assert sorted(type.value for type in toxicity.types) == sorted(types)def test_toxicity_all_types_default(self):- toxicity = Toxicity()+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, )assert sorted(type.value for type in toxicity.types) == sorted(type.value for type in ToxicityType)def test_toxicity_profanity(self):types = ["profanity"]- toxicity = Toxicity(types=types)+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)assert sorted(type.value for type in toxicity.types) == sorted(types)def test_toxicity_insults(self):types = ["insults"]- toxicity = Toxicity(types=types)+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)assert sorted(type.value for type in toxicity.types) == sorted(types)def test_toxicity_threats(self):types = ["threats"]- toxicity = Toxicity(types=types)+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)assert sorted(type.value for type in toxicity.types) == sorted(types)def test_toxicity_mockery(self):types = ["mockery"]- toxicity = Toxicity(types=types)+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)assert sorted(type.value for type in toxicity.types) == sorted(types)def test_toxicity_all_types_invalid(self):types = ["profanity", "insults", "threats", "mockery", "invalid"]with pytest.raises(ValueError):- Toxicity(types=types)+ Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)