Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

Updated tests for vulnerabilities

A-Vamshi14ef77ced8a6
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index 49730a2c..91d3d5d7 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -1,8 +1,10 @@
-from typing import List, Optional
+from typing import List, Optional, Union
from enum import Enum
+from deepeval.models import DeepEvalBaseLLM
+
from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.metrics import BaseRedTeamingMetric
+from deepteam.metrics import BaseRedTeamingMetric, HarmMetric
class CustomVulnerability(BaseVulnerability):
@@ -17,6 +19,9 @@ class CustomVulnerability(BaseVulnerability):
types: Optional[List[str]] = None,
custom_prompt: Optional[str] = None,
metric: Optional[BaseRedTeamingMetric] = None,
+ model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+ async_mode: bool = True,
+ verbose_mode: bool = False,
):
self.name = name
@@ -26,8 +31,19 @@ class CustomVulnerability(BaseVulnerability):
)
self.custom_prompt = custom_prompt
- self.metric = metric
self.criteria = criteria.strip()
+ self.model = model
+ self.async_mode = async_mode
+ self.verbose_mode = verbose_mode
+ if metric:
+ self.metric = metric
+ else:
+ self.metric = HarmMetric(
+ harm_category=self.criteria,
+ model=self.model,
+ async_mode=self.async_mode,
+ verbose_mode=self.verbose_mode
+ )
super().__init__(self.types)
def get_name(self) -> str:
diff --git a/tests/test_core/test_red_teaming.py b/tests/test_core/test_red_teaming.py
index 7610558f..c8f9e1cb 100644
--- a/tests/test_core/test_red_teaming.py
+++ b/tests/test_core/test_red_teaming.py
@@ -9,6 +9,13 @@ async def model_callback(input: str) -> str:
def test_red_teaming():
+ PURPOSE="purpose"
+ TOXICITY="category"
+ GRAPHIC="category"
+ SAFETY="category"
+ MISINFORMATION="category"
+ ILLEGAL="category"
+
custom_vulnerability = CustomVulnerability(
criteria="Criteria",
name="Name",
@@ -17,28 +24,28 @@ def test_red_teaming():
)
vulnerabilities = [
- Bias(),
- Toxicity(),
- Misinformation(),
- IllegalActivity(),
- PromptLeakage(),
- PIILeakage(),
- BFLA(),
+ Bias(purpose=PURPOSE),
+ Toxicity(toxicity_category=TOXICITY),
+ Misinformation(misinformation_category=MISINFORMATION),
+ IllegalActivity(illegal_category=ILLEGAL),
+ PromptLeakage(purpose=PURPOSE),
+ PIILeakage(purpose=PURPOSE),
+ BFLA(purpose=PURPOSE),
BOLA(),
- RBAC(),
+ RBAC(purpose=PURPOSE),
DebugAccess(),
ShellInjection(),
SQLInjection(),
- SSRF(),
- IntellectualProperty(),
- Competition(),
- GraphicContent(),
- PersonalSafety(),
+ SSRF(purpose=PURPOSE),
+ IntellectualProperty(purpose=PURPOSE),
+ Competition(purpose=PURPOSE),
+ GraphicContent(graphic_category=GRAPHIC),
+ PersonalSafety(safety_category=SAFETY),
# Agentic vulnerabilities
- GoalTheft(),
- RecursiveHijacking(),
- Robustness(),
- ExcessiveAgency(),
+ GoalTheft(purpose=PURPOSE),
+ RecursiveHijacking(purpose=PURPOSE),
+ Robustness(purpose=PURPOSE),
+ ExcessiveAgency(purpose=PURPOSE),
custom_vulnerability,
]
diff --git a/tests/test_core/test_vulnerabilities/test_all.py b/tests/test_core/test_vulnerabilities/test_all.py
index 6ea921e4..b0d3b0d8 100644
--- a/tests/test_core/test_vulnerabilities/test_all.py
+++ b/tests/test_core/test_vulnerabilities/test_all.py
@@ -8,6 +8,13 @@ async def model_callback(input: str) -> str:
def test_red_teaming():
+ PURPOSE="purpose"
+ TOXICITY="category"
+ GRAPHIC="category"
+ SAFETY="category"
+ MISINFORMATION="category"
+ ILLEGAL="category"
+
custom_vulnerability = CustomVulnerability(
criteria="Criteria",
name="Name",
@@ -16,28 +23,28 @@ def test_red_teaming():
)
vulnerabilities = [
- Bias(),
- Toxicity(),
- Misinformation(),
- IllegalActivity(),
- PromptLeakage(),
- PIILeakage(),
- BFLA(),
+ Bias(purpose=PURPOSE),
+ Toxicity(toxicity_category=TOXICITY),
+ Misinformation(misinformation_category=MISINFORMATION),
+ IllegalActivity(illegal_category=ILLEGAL),
+ PromptLeakage(purpose=PURPOSE),
+ PIILeakage(purpose=PURPOSE),
+ BFLA(purpose=PURPOSE),
BOLA(),
- RBAC(),
+ RBAC(purpose=PURPOSE),
DebugAccess(),
ShellInjection(),
SQLInjection(),
- SSRF(),
- IntellectualProperty(),
- Competition(),
- GraphicContent(),
- PersonalSafety(),
+ SSRF(purpose=PURPOSE),
+ IntellectualProperty(purpose=PURPOSE),
+ Competition(purpose=PURPOSE),
+ GraphicContent(graphic_category=GRAPHIC),
+ PersonalSafety(safety_category=SAFETY),
# Agentic vulnerabilities
- GoalTheft(),
- RecursiveHijacking(),
- Robustness(),
- ExcessiveAgency(),
+ GoalTheft(purpose=PURPOSE),
+ RecursiveHijacking(purpose=PURPOSE),
+ Robustness(purpose=PURPOSE),
+ ExcessiveAgency(purpose=PURPOSE),
custom_vulnerability,
]
diff --git a/tests/test_core/test_vulnerabilities/test_bfla.py b/tests/test_core/test_vulnerabilities/test_bfla.py
index c889f46d..290ee727 100644
--- a/tests/test_core/test_vulnerabilities/test_bfla.py
+++ b/tests/test_core/test_vulnerabilities/test_bfla.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.bfla import BFLAType
class TestBFLA:
+ PURPOSE="purpose"
def test_bfla_all_types(self):
types = [
@@ -12,28 +13,28 @@ class TestBFLA:
"function_bypass",
"authorization_bypass",
]
- bfla = BFLA(types=types)
+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)
assert sorted(type.value for type in bfla.types) == sorted(types)
def test_bfla_all_types_default(self):
- bfla = BFLA()
+ bfla = BFLA(purpose=TestBFLA.PURPOSE, )
assert sorted(type.value for type in bfla.types) == sorted(
type.value for type in BFLAType
)
def test_bfla_privilege_escalation(self):
types = ["privilege_escalation"]
- bfla = BFLA(types=types)
+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)
assert sorted(type.value for type in bfla.types) == sorted(types)
def test_bfla_function_bypass(self):
types = ["function_bypass"]
- bfla = BFLA(types=types)
+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)
assert sorted(type.value for type in bfla.types) == sorted(types)
def test_bfla_authorization_bypass(self):
types = ["authorization_bypass"]
- bfla = BFLA(types=types)
+ bfla = BFLA(purpose=TestBFLA.PURPOSE, types=types)
assert sorted(type.value for type in bfla.types) == sorted(types)
def test_bfla_all_types_invalid(self):
@@ -44,4 +45,4 @@ class TestBFLA:
"invalid",
]
with pytest.raises(ValueError):
- BFLA(types=types)
+ BFLA(purpose=TestBFLA.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_bias.py b/tests/test_core/test_vulnerabilities/test_bias.py
index fe792d01..ac4a678a 100644
--- a/tests/test_core/test_vulnerabilities/test_bias.py
+++ b/tests/test_core/test_vulnerabilities/test_bias.py
@@ -5,39 +5,40 @@ from deepteam.vulnerabilities.bias import BiasType
class TestBias:
+ PURPOSE="purpose"
def test_bias_all_types(self):
types = ["gender", "politics", "religion", "race"]
- bias = Bias(types=types)
+ bias = Bias(purpose=TestBias.PURPOSE, types=types)
assert sorted(type.value for type in bias.types) == sorted(types)
def test_bias_all_types_default(self):
- bias = Bias()
+ bias = Bias(purpose=TestBias.PURPOSE, )
assert sorted(type.value for type in bias.types) == sorted(
type.value for type in BiasType
)
def test_bias_gender(self):
types = ["gender"]
- bias = Bias(types=types)
+ bias = Bias(purpose=TestBias.PURPOSE, types=types)
assert sorted(type.value for type in bias.types) == sorted(types)
def test_bias_politics(self):
types = ["politics"]
- bias = Bias(types=types)
+ bias = Bias(purpose=TestBias.PURPOSE, types=types)
assert sorted(type.value for type in bias.types) == sorted(types)
def test_bias_religion(self):
types = ["religion"]
- bias = Bias(types=types)
+ bias = Bias(purpose=TestBias.PURPOSE, types=types)
assert sorted(type.value for type in bias.types) == sorted(types)
def test_bias_race(self):
types = ["race"]
- bias = Bias(types=types)
+ bias = Bias(purpose=TestBias.PURPOSE, types=types)
assert sorted(type.value for type in bias.types) == sorted(types)
def test_bias_all_types_invalid(self):
types = ["gender", "politics", "religion", "race", "invalid"]
with pytest.raises(ValueError):
- Bias(types=types)
+ Bias(purpose=TestBias.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_competition.py b/tests/test_core/test_vulnerabilities/test_competition.py
index 741b2529..4eb21182 100644
--- a/tests/test_core/test_vulnerabilities/test_competition.py
+++ b/tests/test_core/test_vulnerabilities/test_competition.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.competition import CompetitionType
class TestCompetition:
+ PURPOSE="purpose"
def test_competition_all_types(self):
types = [
@@ -13,33 +14,33 @@ class TestCompetition:
"discreditation",
"confidential strategies",
]
- competition = Competition(types=types)
+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)
assert sorted(type.value for type in competition.types) == sorted(types)
def test_competition_all_types_default(self):
- competition = Competition()
+ competition = Competition(purpose=TestCompetition.PURPOSE, )
assert sorted(type.value for type in competition.types) == sorted(
type.value for type in CompetitionType
)
def test_competition_competitor_mention(self):
types = ["competitor mention"]
- competition = Competition(types=types)
+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)
assert sorted(type.value for type in competition.types) == sorted(types)
def test_competition_market_manipulation(self):
types = ["market manipulation"]
- competition = Competition(types=types)
+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)
assert sorted(type.value for type in competition.types) == sorted(types)
def test_competition_discreditation(self):
types = ["discreditation"]
- competition = Competition(types=types)
+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)
assert sorted(type.value for type in competition.types) == sorted(types)
def test_competition_confidential_strategies(self):
types = ["confidential strategies"]
- competition = Competition(types=types)
+ competition = Competition(purpose=TestCompetition.PURPOSE, types=types)
assert sorted(type.value for type in competition.types) == sorted(types)
def test_competition_all_types_invalid(self):
@@ -51,4 +52,4 @@ class TestCompetition:
"invalid",
]
with pytest.raises(ValueError):
- Competition(types=types)
+ Competition(purpose=TestCompetition.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py b/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py
index d78447d6..626e2dd8 100644
--- a/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py
+++ b/tests/test_core/test_vulnerabilities/test_custom_vulnerability.py
@@ -15,6 +15,7 @@ class TestCustomVulnerability:
assert custom_vulnerability.criteria == "Criteria"
assert custom_vulnerability.name == "Name"
assert custom_vulnerability.custom_prompt == "Custom prompt"
+ assert custom_vulnerability.metric is not None
assert sorted(
type.value for type in custom_vulnerability.types
) == sorted(["type1", "type2"])
diff --git a/tests/test_core/test_vulnerabilities/test_excessive_agency.py b/tests/test_core/test_vulnerabilities/test_excessive_agency.py
index 1131af0f..50243d65 100644
--- a/tests/test_core/test_vulnerabilities/test_excessive_agency.py
+++ b/tests/test_core/test_vulnerabilities/test_excessive_agency.py
@@ -7,37 +7,38 @@ from deepteam.vulnerabilities.agentic.excessive_agency import (
class TestExcessiveAgency:
+ PURPOSE="purpose"
def test_excessive_agency_all_types(self):
types = ["functionality", "permissions", "autonomy"]
- excessive_agency = ExcessiveAgency(types=types)
+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)
assert sorted(type.value for type in excessive_agency.types) == sorted(
types
)
def test_excessive_agency_all_types_default(self):
- excessive_agency = ExcessiveAgency()
+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, )
assert sorted(type.value for type in excessive_agency.types) == sorted(
type.value for type in ExcessiveAgencyType
)
def test_excessive_agency_functionality(self):
types = ["functionality"]
- excessive_agency = ExcessiveAgency(types=types)
+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)
assert sorted(type.value for type in excessive_agency.types) == sorted(
types
)
def test_excessive_agency_permissions(self):
types = ["permissions"]
- excessive_agency = ExcessiveAgency(types=types)
+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)
assert sorted(type.value for type in excessive_agency.types) == sorted(
types
)
def test_excessive_agency_autonomy(self):
types = ["autonomy"]
- excessive_agency = ExcessiveAgency(types=types)
+ excessive_agency = ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)
assert sorted(type.value for type in excessive_agency.types) == sorted(
types
)
@@ -45,4 +46,4 @@ class TestExcessiveAgency:
def test_excessive_agency_all_types_invalid(self):
types = ["functionality", "permissions", "autonomy", "invalid"]
with pytest.raises(ValueError):
- ExcessiveAgency(types=types)
+ ExcessiveAgency(purpose=TestExcessiveAgency.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_goal_theft.py b/tests/test_core/test_vulnerabilities/test_goal_theft.py
index 0b4d09b4..2e0af11b 100644
--- a/tests/test_core/test_vulnerabilities/test_goal_theft.py
+++ b/tests/test_core/test_vulnerabilities/test_goal_theft.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.agentic.goal_theft import GoalTheftType
class TestGoalTheft:
+ PURPOSE="purpose"
def test_goal_theft_all_types(self):
types = [
@@ -12,28 +13,28 @@ class TestGoalTheft:
"cooperative_dialogue",
"social_engineering",
]
- goal_theft = GoalTheft(types=types)
+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)
assert sorted(type.value for type in goal_theft.types) == sorted(types)
def test_goal_theft_all_types_default(self):
- goal_theft = GoalTheft()
+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, )
assert sorted(type.value for type in goal_theft.types) == sorted(
type.value for type in GoalTheftType
)
def test_goal_theft_escalating_probing(self):
types = ["escalating_probing"]
- goal_theft = GoalTheft(types=types)
+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)
assert sorted(type.value for type in goal_theft.types) == sorted(types)
def test_goal_theft_cooperative_dialogue(self):
types = ["cooperative_dialogue"]
- goal_theft = GoalTheft(types=types)
+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)
assert sorted(type.value for type in goal_theft.types) == sorted(types)
def test_goal_theft_social_engineering(self):
types = ["social_engineering"]
- goal_theft = GoalTheft(types=types)
+ goal_theft = GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)
assert sorted(type.value for type in goal_theft.types) == sorted(types)
def test_goal_theft_all_types_invalid(self):
@@ -44,4 +45,4 @@ class TestGoalTheft:
"invalid",
]
with pytest.raises(ValueError):
- GoalTheft(types=types)
+ GoalTheft(purpose=TestGoalTheft.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_graphic_content.py b/tests/test_core/test_vulnerabilities/test_graphic_content.py
index 16250ed1..db17aa53 100644
--- a/tests/test_core/test_vulnerabilities/test_graphic_content.py
+++ b/tests/test_core/test_vulnerabilities/test_graphic_content.py
@@ -5,37 +5,38 @@ from deepteam.vulnerabilities.graphic_content import GraphicContentType
class TestGraphicContent:
+ GRAPHIC="category"
def test_graphic_content_all_types(self):
types = ["sexual content", "graphic content", "pornographic content"]
- graphic_content = GraphicContent(types=types)
+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)
assert sorted(type.value for type in graphic_content.types) == sorted(
types
)
def test_graphic_content_all_types_default(self):
- graphic_content = GraphicContent()
+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, )
assert sorted(type.value for type in graphic_content.types) == sorted(
type.value for type in GraphicContentType
)
def test_graphic_content_sexual_content(self):
types = ["sexual content"]
- graphic_content = GraphicContent(types=types)
+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)
assert sorted(type.value for type in graphic_content.types) == sorted(
types
)
def test_graphic_content_graphic_content(self):
types = ["graphic content"]
- graphic_content = GraphicContent(types=types)
+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)
assert sorted(type.value for type in graphic_content.types) == sorted(
types
)
def test_graphic_content_pornographic_content(self):
types = ["pornographic content"]
- graphic_content = GraphicContent(types=types)
+ graphic_content = GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)
assert sorted(type.value for type in graphic_content.types) == sorted(
types
)
@@ -48,4 +49,4 @@ class TestGraphicContent:
"invalid",
]
with pytest.raises(ValueError):
- GraphicContent(types=types)
+ GraphicContent(graphic_category=TestGraphicContent.GRAPHIC, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_illegal_activity.py b/tests/test_core/test_vulnerabilities/test_illegal_activity.py
index 1ce14eb1..5c78815e 100644
--- a/tests/test_core/test_vulnerabilities/test_illegal_activity.py
+++ b/tests/test_core/test_vulnerabilities/test_illegal_activity.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.illegal_activity import IllegalActivityType
class TestIllegalActivity:
+ ILLEGAL="category"
def test_illegal_activity_all_types(self):
types = [
@@ -16,62 +17,62 @@ class TestIllegalActivity:
"cybercrime",
"child exploitation",
]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_all_types_default(self):
- illegal_activity = IllegalActivity()
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, )
assert sorted(type.value for type in illegal_activity.types) == sorted(
type.value for type in IllegalActivityType
)
def test_illegal_activity_weapons(self):
types = ["weapons"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_illegal_drugs(self):
types = ["illegal drugs"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_violent_crimes(self):
types = ["violent crimes"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_non_violent_crimes(self):
types = ["non violent crimes"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_sex_crimes(self):
types = ["sex crimes"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_cybercrime(self):
types = ["cybercrime"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
def test_illegal_activity_child_exploitation(self):
types = ["child exploitation"]
- illegal_activity = IllegalActivity(types=types)
+ illegal_activity = IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
assert sorted(type.value for type in illegal_activity.types) == sorted(
types
)
@@ -88,4 +89,4 @@ class TestIllegalActivity:
"invalid",
]
with pytest.raises(ValueError):
- IllegalActivity(types=types)
+ IllegalActivity(illegal_category=TestIllegalActivity.ILLEGAL, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_intellectual_property.py b/tests/test_core/test_vulnerabilities/test_intellectual_property.py
index 2c259817..502029f3 100644
--- a/tests/test_core/test_vulnerabilities/test_intellectual_property.py
+++ b/tests/test_core/test_vulnerabilities/test_intellectual_property.py
@@ -7,6 +7,7 @@ from deepteam.vulnerabilities.intellectual_property import (
class TestIntellectualProperty:
+ PURPOSE="purpose"
def test_intellectual_property_all_types(self):
types = [
@@ -15,41 +16,41 @@ class TestIntellectualProperty:
"trademark infringement",
"patent disclosure",
]
- intellectual_property = IntellectualProperty(types=types)
+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)
assert sorted(
type.value for type in intellectual_property.types
) == sorted(types)
def test_intellectual_property_all_types_default(self):
- intellectual_property = IntellectualProperty()
+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, )
assert sorted(
type.value for type in intellectual_property.types
) == sorted(type.value for type in IntellectualPropertyType)
def test_intellectual_property_imitation(self):
types = ["imitation"]
- intellectual_property = IntellectualProperty(types=types)
+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)
assert sorted(
type.value for type in intellectual_property.types
) == sorted(types)
def test_intellectual_property_copyright_violations(self):
types = ["copyright violations"]
- intellectual_property = IntellectualProperty(types=types)
+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)
assert sorted(
type.value for type in intellectual_property.types
) == sorted(types)
def test_intellectual_property_trademark_infringement(self):
types = ["trademark infringement"]
- intellectual_property = IntellectualProperty(types=types)
+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)
assert sorted(
type.value for type in intellectual_property.types
) == sorted(types)
def test_intellectual_property_patent_disclosure(self):
types = ["patent disclosure"]
- intellectual_property = IntellectualProperty(types=types)
+ intellectual_property = IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)
assert sorted(
type.value for type in intellectual_property.types
) == sorted(types)
@@ -63,4 +64,4 @@ class TestIntellectualProperty:
"invalid",
]
with pytest.raises(ValueError):
- IntellectualProperty(types=types)
+ IntellectualProperty(purpose=TestIntellectualProperty.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_misinformation.py b/tests/test_core/test_vulnerabilities/test_misinformation.py
index 66225b04..523c92fb 100644
--- a/tests/test_core/test_vulnerabilities/test_misinformation.py
+++ b/tests/test_core/test_vulnerabilities/test_misinformation.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.misinformation import MisinformationType
class TestMisinformation:
+ MISINFORMATION="category"
def test_misinformation_all_types(self):
types = [
@@ -12,34 +13,34 @@ class TestMisinformation:
"unsupported claims",
"expertize misrepresentation",
]
- misinformation = Misinformation(types=types)
+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)
assert sorted(type.value for type in misinformation.types) == sorted(
types
)
def test_misinformation_all_types_default(self):
- misinformation = Misinformation()
+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, )
assert sorted(type.value for type in misinformation.types) == sorted(
type.value for type in MisinformationType
)
def test_misinformation_factual_errors(self):
types = ["factual errors"]
- misinformation = Misinformation(types=types)
+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)
assert sorted(type.value for type in misinformation.types) == sorted(
types
)
def test_misinformation_unsupported_claims(self):
types = ["unsupported claims"]
- misinformation = Misinformation(types=types)
+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)
assert sorted(type.value for type in misinformation.types) == sorted(
types
)
def test_misinformation_expertize_misrepresentation(self):
types = ["expertize misrepresentation"]
- misinformation = Misinformation(types=types)
+ misinformation = Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)
assert sorted(type.value for type in misinformation.types) == sorted(
types
)
@@ -52,4 +53,4 @@ class TestMisinformation:
"invalid",
]
with pytest.raises(ValueError):
- Misinformation(types=types)
+ Misinformation(misinformation_category=TestMisinformation.MISINFORMATION, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_personal_safety.py b/tests/test_core/test_vulnerabilities/test_personal_safety.py
index 6cec5652..b2dbcbcf 100644
--- a/tests/test_core/test_vulnerabilities/test_personal_safety.py
+++ b/tests/test_core/test_vulnerabilities/test_personal_safety.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.personal_safety import PersonalSafetyType
class TestPersonalSafety:
+ SAFETY="category"
def test_personal_safety_all_types(self):
types = [
@@ -14,48 +15,48 @@ class TestPersonalSafety:
"dangerous challenges",
"stalking",
]
- personal_safety = PersonalSafety(types=types)
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
assert sorted(type.value for type in personal_safety.types) == sorted(
types
)
def test_personal_safety_all_types_default(self):
- personal_safety = PersonalSafety()
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, )
assert sorted(type.value for type in personal_safety.types) == sorted(
type.value for type in PersonalSafetyType
)
def test_personal_safety_bullying(self):
types = ["bullying"]
- personal_safety = PersonalSafety(types=types)
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
assert sorted(type.value for type in personal_safety.types) == sorted(
types
)
def test_personal_safety_self_harm(self):
types = ["self-harm"]
- personal_safety = PersonalSafety(types=types)
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
assert sorted(type.value for type in personal_safety.types) == sorted(
types
)
def test_personal_safety_unsafe_practices(self):
types = ["unsafe practices"]
- personal_safety = PersonalSafety(types=types)
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
assert sorted(type.value for type in personal_safety.types) == sorted(
types
)
def test_personal_safety_dangerous_challenges(self):
types = ["dangerous challenges"]
- personal_safety = PersonalSafety(types=types)
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
assert sorted(type.value for type in personal_safety.types) == sorted(
types
)
def test_personal_safety_stalking(self):
types = ["stalking"]
- personal_safety = PersonalSafety(types=types)
+ personal_safety = PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
assert sorted(type.value for type in personal_safety.types) == sorted(
types
)
@@ -70,4 +71,4 @@ class TestPersonalSafety:
"invalid",
]
with pytest.raises(ValueError):
- PersonalSafety(types=types)
+ PersonalSafety(safety_category=TestPersonalSafety.SAFETY, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_pii_leakage.py b/tests/test_core/test_vulnerabilities/test_pii_leakage.py
index dc1b6c49..c160afea 100644
--- a/tests/test_core/test_vulnerabilities/test_pii_leakage.py
+++ b/tests/test_core/test_vulnerabilities/test_pii_leakage.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.pii_leakage import PIILeakageType
class TestPIILeakage:
+ PURPOSE="purpose"
def test_pii_leakage_all_types(self):
types = [
@@ -13,33 +14,33 @@ class TestPIILeakage:
"session leak",
"social manipulation",
]
- pii_leakage = PIILeakage(types=types)
+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)
assert sorted(type.value for type in pii_leakage.types) == sorted(types)
def test_pii_leakage_all_types_default(self):
- pii_leakage = PIILeakage()
+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, )
assert sorted(type.value for type in pii_leakage.types) == sorted(
type.value for type in PIILeakageType
)
def test_pii_leakage_database_access(self):
types = ["api and database access"]
- pii_leakage = PIILeakage(types=types)
+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)
assert sorted(type.value for type in pii_leakage.types) == sorted(types)
def test_pii_leakage_direct_disclosure(self):
types = ["direct disclosure"]
- pii_leakage = PIILeakage(types=types)
+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)
assert sorted(type.value for type in pii_leakage.types) == sorted(types)
def test_pii_leakage_session_leak(self):
types = ["session leak"]
- pii_leakage = PIILeakage(types=types)
+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)
assert sorted(type.value for type in pii_leakage.types) == sorted(types)
def test_pii_leakage_social_manipulation(self):
types = ["social manipulation"]
- pii_leakage = PIILeakage(types=types)
+ pii_leakage = PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)
assert sorted(type.value for type in pii_leakage.types) == sorted(types)
def test_pii_leakage_all_types_invalid(self):
@@ -51,4 +52,4 @@ class TestPIILeakage:
"invalid",
]
with pytest.raises(ValueError):
- PIILeakage(types=types)
+ PIILeakage(purpose=TestPIILeakage.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_prompt_leakage.py b/tests/test_core/test_vulnerabilities/test_prompt_leakage.py
index 0ee7ab5d..bdbec163 100644
--- a/tests/test_core/test_vulnerabilities/test_prompt_leakage.py
+++ b/tests/test_core/test_vulnerabilities/test_prompt_leakage.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.prompt_leakage import PromptLeakageType
class TestPromptLeakage:
+ PURPOSE="purpose"
def test_prompt_leakage_all_types(self):
types = [
@@ -13,41 +14,41 @@ class TestPromptLeakage:
"guard exposure",
"permissions and roles",
]
- prompt_leakage = PromptLeakage(types=types)
+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)
assert sorted(type.value for type in prompt_leakage.types) == sorted(
types
)
def test_prompt_leakage_all_types_default(self):
- prompt_leakage = PromptLeakage()
+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, )
assert sorted(type.value for type in prompt_leakage.types) == sorted(
type.value for type in PromptLeakageType
)
def test_prompt_leakage_secrets_and_credentials(self):
types = ["secrets and credentials"]
- prompt_leakage = PromptLeakage(types=types)
+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)
assert sorted(type.value for type in prompt_leakage.types) == sorted(
types
)
def test_prompt_leakage_instructions(self):
types = ["instructions"]
- prompt_leakage = PromptLeakage(types=types)
+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)
assert sorted(type.value for type in prompt_leakage.types) == sorted(
types
)
def test_prompt_leakage_guard_exposure(self):
types = ["guard exposure"]
- prompt_leakage = PromptLeakage(types=types)
+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)
assert sorted(type.value for type in prompt_leakage.types) == sorted(
types
)
def test_prompt_leakage_permissions_and_roles(self):
types = ["permissions and roles"]
- prompt_leakage = PromptLeakage(types=types)
+ prompt_leakage = PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)
assert sorted(type.value for type in prompt_leakage.types) == sorted(
types
)
@@ -61,4 +62,4 @@ class TestPromptLeakage:
"invalid",
]
with pytest.raises(ValueError):
- PromptLeakage(types=types)
+ PromptLeakage(purpose=TestPromptLeakage.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_rbac.py b/tests/test_core/test_vulnerabilities/test_rbac.py
index 7e51a87e..a4919393 100644
--- a/tests/test_core/test_vulnerabilities/test_rbac.py
+++ b/tests/test_core/test_vulnerabilities/test_rbac.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.rbac import RBACType
class TestRBAC:
+ PURPOSE="purpose"
def test_rbac_all_types(self):
types = [
@@ -12,28 +13,28 @@ class TestRBAC:
"privilege escalation",
"unauthorized role assumption",
]
- rbac = RBAC(types=types)
+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)
assert sorted(type.value for type in rbac.types) == sorted(types)
def test_rbac_all_types_default(self):
- rbac = RBAC()
+ rbac = RBAC(purpose=TestRBAC.PURPOSE, )
assert sorted(type.value for type in rbac.types) == sorted(
type.value for type in RBACType
)
def test_rbac_role_bypass(self):
types = ["role bypass"]
- rbac = RBAC(types=types)
+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)
assert sorted(type.value for type in rbac.types) == sorted(types)
def test_rbac_privilege_escalation(self):
types = ["privilege escalation"]
- rbac = RBAC(types=types)
+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)
assert sorted(type.value for type in rbac.types) == sorted(types)
def test_rbac_unauthorized_role_assumption(self):
types = ["unauthorized role assumption"]
- rbac = RBAC(types=types)
+ rbac = RBAC(purpose=TestRBAC.PURPOSE, types=types)
assert sorted(type.value for type in rbac.types) == sorted(types)
def test_rbac_all_types_invalid(self):
@@ -44,4 +45,4 @@ class TestRBAC:
"invalid",
]
with pytest.raises(ValueError):
- RBAC(types=types)
+ RBAC(purpose=TestRBAC.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py b/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py
index a29017be..91752a9a 100644
--- a/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py
+++ b/tests/test_core/test_vulnerabilities/test_recursive_hijacking.py
@@ -7,6 +7,7 @@ from deepteam.vulnerabilities.agentic.recursive_hijacking import (
class TestRecursiveHijacking:
+ PURPOSE="purpose"
def test_recursive_hijacking_all_types(self):
types = [
@@ -14,34 +15,34 @@ class TestRecursiveHijacking:
"recursive_objective_chaining",
"goal_propagation_attacks",
]
- recursive_hijacking = RecursiveHijacking(types=types)
+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)
assert sorted(
type.value for type in recursive_hijacking.types
) == sorted(types)
def test_recursive_hijacking_all_types_default(self):
- recursive_hijacking = RecursiveHijacking()
+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, )
assert sorted(
type.value for type in recursive_hijacking.types
) == sorted(type.value for type in RecursiveHijackingType)
def test_recursive_hijacking_self_modifying_goals(self):
types = ["self_modifying_goals"]
- recursive_hijacking = RecursiveHijacking(types=types)
+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)
assert sorted(
type.value for type in recursive_hijacking.types
) == sorted(types)
def test_recursive_hijacking_recursive_objective_chaining(self):
types = ["recursive_objective_chaining"]
- recursive_hijacking = RecursiveHijacking(types=types)
+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)
assert sorted(
type.value for type in recursive_hijacking.types
) == sorted(types)
def test_recursive_hijacking_goal_propagation_attacks(self):
types = ["goal_propagation_attacks"]
- recursive_hijacking = RecursiveHijacking(types=types)
+ recursive_hijacking = RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)
assert sorted(
type.value for type in recursive_hijacking.types
) == sorted(types)
@@ -54,4 +55,4 @@ class TestRecursiveHijacking:
"invalid",
]
with pytest.raises(ValueError):
- RecursiveHijacking(types=types)
+ RecursiveHijacking(purpose=TestRecursiveHijacking.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_robustness.py b/tests/test_core/test_vulnerabilities/test_robustness.py
index 70bf827d..26378382 100644
--- a/tests/test_core/test_vulnerabilities/test_robustness.py
+++ b/tests/test_core/test_vulnerabilities/test_robustness.py
@@ -5,29 +5,30 @@ from deepteam.vulnerabilities.agentic.robustness import RobustnessType
class TestRobustness:
+ PURPOSE="purpose"
def test_robustness_all_types(self):
types = ["input overreliance", "hijacking"]
- robustness = Robustness(types=types)
+ robustness = Robustness(purpose=TestRobustness.PURPOSE, types=types)
assert sorted(type.value for type in robustness.types) == sorted(types)
def test_robustness_all_types_default(self):
- robustness = Robustness()
+ robustness = Robustness(purpose=TestRobustness.PURPOSE, )
assert sorted(type.value for type in robustness.types) == sorted(
type.value for type in RobustnessType
)
def test_robustness_input_overreliance(self):
types = ["input overreliance"]
- robustness = Robustness(types=types)
+ robustness = Robustness(purpose=TestRobustness.PURPOSE, types=types)
assert sorted(type.value for type in robustness.types) == sorted(types)
def test_robustness_hijacking(self):
types = ["hijacking"]
- robustness = Robustness(types=types)
+ robustness = Robustness(purpose=TestRobustness.PURPOSE, types=types)
assert sorted(type.value for type in robustness.types) == sorted(types)
def test_robustness_all_types_invalid(self):
types = ["input overreliance", "hijacking", "invalid"]
with pytest.raises(ValueError):
- Robustness(types=types)
+ Robustness(purpose=TestRobustness.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_ssrf.py b/tests/test_core/test_vulnerabilities/test_ssrf.py
index d655fb8c..f810a262 100644
--- a/tests/test_core/test_vulnerabilities/test_ssrf.py
+++ b/tests/test_core/test_vulnerabilities/test_ssrf.py
@@ -5,6 +5,7 @@ from deepteam.vulnerabilities.ssrf import SSRFType
class TestSSRF:
+ PURPOSE="purpose"
def test_ssrf_all_types(self):
types = [
@@ -12,28 +13,28 @@ class TestSSRF:
"cloud_metadata_access",
"port_scanning",
]
- ssrf = SSRF(types=types)
+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)
assert sorted(type.value for type in ssrf.types) == sorted(types)
def test_ssrf_all_types_default(self):
- ssrf = SSRF()
+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, )
assert sorted(type.value for type in ssrf.types) == sorted(
type.value for type in SSRFType
)
def test_ssrf_internal_service_access(self):
types = ["internal_service_access"]
- ssrf = SSRF(types=types)
+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)
assert sorted(type.value for type in ssrf.types) == sorted(types)
def test_ssrf_cloud_metadata_access(self):
types = ["cloud_metadata_access"]
- ssrf = SSRF(types=types)
+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)
assert sorted(type.value for type in ssrf.types) == sorted(types)
def test_ssrf_port_scanning(self):
types = ["port_scanning"]
- ssrf = SSRF(types=types)
+ ssrf = SSRF(purpose=TestSSRF.PURPOSE, types=types)
assert sorted(type.value for type in ssrf.types) == sorted(types)
def test_ssrf_all_types_invalid(self):
@@ -44,4 +45,4 @@ class TestSSRF:
"invalid",
]
with pytest.raises(ValueError):
- SSRF(types=types)
+ SSRF(purpose=TestSSRF.PURPOSE, types=types)
diff --git a/tests/test_core/test_vulnerabilities/test_toxicity.py b/tests/test_core/test_vulnerabilities/test_toxicity.py
index 5fa9d482..874afb5e 100644
--- a/tests/test_core/test_vulnerabilities/test_toxicity.py
+++ b/tests/test_core/test_vulnerabilities/test_toxicity.py
@@ -5,39 +5,40 @@ from deepteam.vulnerabilities.toxicity import ToxicityType
class TestToxicity:
+ TOXICITY="category"
def test_toxicity_all_types(self):
types = ["profanity", "insults", "threats", "mockery"]
- toxicity = Toxicity(types=types)
+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)
assert sorted(type.value for type in toxicity.types) == sorted(types)
def test_toxicity_all_types_default(self):
- toxicity = Toxicity()
+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, )
assert sorted(type.value for type in toxicity.types) == sorted(
type.value for type in ToxicityType
)
def test_toxicity_profanity(self):
types = ["profanity"]
- toxicity = Toxicity(types=types)
+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)
assert sorted(type.value for type in toxicity.types) == sorted(types)
def test_toxicity_insults(self):
types = ["insults"]
- toxicity = Toxicity(types=types)
+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)
assert sorted(type.value for type in toxicity.types) == sorted(types)
def test_toxicity_threats(self):
types = ["threats"]
- toxicity = Toxicity(types=types)
+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)
assert sorted(type.value for type in toxicity.types) == sorted(types)
def test_toxicity_mockery(self):
types = ["mockery"]
- toxicity = Toxicity(types=types)
+ toxicity = Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)
assert sorted(type.value for type in toxicity.types) == sorted(types)
def test_toxicity_all_types_invalid(self):
types = ["profanity", "insults", "threats", "mockery", "invalid"]
with pytest.raises(ValueError):
- Toxicity(types=types)
+ Toxicity(toxicity_category=TestToxicity.TOXICITY, types=types)