DeepTeam is a framework to red team LLMs and AI agents.
.
penguine8a83cb2dc3fb
diff --git a/deepteam/attacks/base_attack.py b/deepteam/attacks/base_attack.pyindex df84f43c..0ed22a7a 100644--- a/deepteam/attacks/base_attack.py+++ b/deepteam/attacks/base_attack.py@@ -4,13 +4,7 @@ from abc import ABC, abstractmethodclass BaseAttack(ABC):weight: int = 1multi_turn: bool = False+ name: str = None # Subclasses should override this- def enhance(self, attack: str, *args, **kwargs) -> str:- pass-- async def a_enhance(self, attack: str, *args, **kwargs) -> str:- return self.enhance(attack, *args, **kwargs)-- @abstractmethoddef get_name(self) -> str:- raise NotImplementedError+ return self.name if self.name else self.__class__.__name__diff --git a/deepteam/attacks/multi_turn/__init__.py b/deepteam/attacks/multi_turn/__init__.pyindex d1d46a64..95163e5f 100644--- a/deepteam/attacks/multi_turn/__init__.py+++ b/deepteam/attacks/multi_turn/__init__.py@@ -11,16 +11,3 @@ __all__ = ["SequentialJailbreak","BadLikertJudge",]--MULTI_TURN_ATTACK_MAP = {- v.__name__: v- for v in [- CrescendoJailbreaking,- LinearJailbreaking,- TreeJailbreaking,- SequentialJailbreak,- BadLikertJudge,- ]-}--MULTI_TURN_ATTACK_NAMES = sorted(list(MULTI_TURN_ATTACK_MAP.keys()))diff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.pyindex 230fa30a..5480f893 100644--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py@@ -4,7 +4,9 @@ from typing import Optional, Union, List, Dictfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_modelimport random-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+)from deepteam.attacks.multi_turn.bad_likert_judge.schema import (BLJJudge,)@@ -18,20 +20,25 @@ from deepteam.attacks.attack_simulator.utils import (from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attackfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.multi_turn.base_schema import NonRefusal+from deepteam.attacks.single_turn.base_single_turn_attack import (+ BaseSingleTurnAttack,+)from deepteam.errors import ModelRefusalErrorfrom deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability-class BadLikertJudge(BaseAttack):+class BadLikertJudge(BaseMultiTurnAttack):+ name = "Bad Likert Judge"+def __init__(self,weight: int = 1,category: str = "bias",num_turns: int = 5,enable_refinement: bool = True,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weight@@ -457,7 +464,7 @@ class BadLikertJudge(BaseAttack):return resultdef get_name(self) -> str:- return "Bad Likert Judge"+ return self.name@classmethoddef get_supported_categories(cls) -> list[str]:diff --git a/deepteam/attacks/multi_turn/base_multi_turn_attack.py b/deepteam/attacks/multi_turn/base_multi_turn_attack.pynew file mode 100644index 00000000..d19df19c--- /dev/null+++ b/deepteam/attacks/multi_turn/base_multi_turn_attack.py@@ -0,0 +1,26 @@+from deepteam.attacks.base_attack import BaseAttack+from deepteam.vulnerabilities.base_vulnerability import BaseVulnerability+from deepteam.attacks.multi_turn.types import CallbackType+from typing import Optional, List, Dict+from deepteam.test_case import RTTurn+from deepteam.vulnerabilities.types import VulnerabilityType+++class BaseMultiTurnAttack(BaseAttack):+ def progress(+ self,+ vulnerability: BaseVulnerability,+ model_callback: CallbackType,+ turns: Optional[List[RTTurn]] = None,+ **kwargs,+ ) -> Dict[VulnerabilityType, List[RTTurn]]:+ pass++ async def a_progress(+ self,+ vulnerability: BaseVulnerability,+ model_callback: CallbackType,+ turns: Optional[List[RTTurn]] = None,+ **kwargs,+ ) -> Dict[VulnerabilityType, List[RTTurn]]:+ passdiff --git a/deepteam/attacks/multi_turn/constants.py b/deepteam/attacks/multi_turn/constants.pynew file mode 100644index 00000000..d016665d--- /dev/null+++ b/deepteam/attacks/multi_turn/constants.py@@ -0,0 +1,23 @@+from typing import Dict++from .base_multi_turn_attack import BaseMultiTurnAttack+from .crescendo_jailbreaking import CrescendoJailbreaking+from .linear_jailbreaking import LinearJailbreaking+from .tree_jailbreaking import TreeJailbreaking+from .sequential_break import SequentialJailbreak+from .bad_likert_judge import BadLikertJudge++MULTI_TURN_ATTACK_CLASSES_MAP: Dict[str, BaseMultiTurnAttack] = {+ v.name: v+ for v in [+ CrescendoJailbreaking,+ LinearJailbreaking,+ TreeJailbreaking,+ SequentialJailbreak,+ BadLikertJudge,+ ]+}++MULTI_TURN_ATTACK_NAMES = sorted(+ v.name for v in MULTI_TURN_ATTACK_CLASSES_MAP.values()+)diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.pyindex 5b0a7708..27e226c8 100644--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py@@ -9,7 +9,9 @@ from deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_modelfrom deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+)from deepteam.attacks.multi_turn.crescendo_jailbreaking.template import (JailBreakingCrescendoTemplate,)@@ -23,6 +25,9 @@ from deepteam.attacks.attack_simulator.utils import (a_generate,)from deepteam.attacks.multi_turn.types import CallbackType+from deepteam.attacks.single_turn.base_single_turn_attack import (+ BaseSingleTurnAttack,+)from deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability@@ -50,14 +55,15 @@ class MemorySystem:return new_conversation_id-class CrescendoJailbreaking(BaseAttack):+class CrescendoJailbreaking(BaseMultiTurnAttack):+ name = "Crescendo Jailbreaking"def __init__(self,weight: int = 1,max_rounds: int = 10,max_backtracks: int = 10,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weight@@ -422,7 +428,7 @@ class CrescendoJailbreaking(BaseAttack):def progress(self,- vulnerability: "BaseVulnerability",+ vulnerability: BaseVulnerability,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,) -> Dict[VulnerabilityType, List[RTTurn]]:@@ -505,7 +511,7 @@ class CrescendoJailbreaking(BaseAttack):async def a_progress(self,- vulnerability: "BaseVulnerability",+ vulnerability: BaseVulnerability,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,) -> Dict[VulnerabilityType, List[RTTurn]]:@@ -778,4 +784,4 @@ class CrescendoJailbreaking(BaseAttack):return eval_response.value, eval_response.metadatadef get_name(self) -> str:- return "Crescendo Jailbreaking"+ return self.namediff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.pyindex ce01515c..f114e909 100644--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py@@ -18,7 +18,9 @@ from deepteam.attacks.attack_simulator.utils import (a_generate,)from deepteam.attacks.multi_turn.types import CallbackType-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+)from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attackfrom deepteam.attacks.multi_turn.base_schema import NonRefusalfrom deepteam.attacks.multi_turn.base_template import BaseMultiTurnTemplate@@ -26,14 +28,19 @@ from deepteam.errors import ModelRefusalErrorfrom deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability+from deepteam.attacks.single_turn.base_single_turn_attack import (+ BaseSingleTurnAttack,+)++class LinearJailbreaking(BaseMultiTurnAttack):+ name = "Linear Jailbreaking"-class LinearJailbreaking(BaseAttack):def __init__(self,weight: int = 1,num_turns: int = 5,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weight@@ -430,4 +437,4 @@ class LinearJailbreaking(BaseAttack):return resultdef get_name(self) -> str:- return "Linear Jailbreaking"+ return self.namediff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.pyindex 654a0faf..aeeadd79 100644--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py@@ -5,7 +5,9 @@ import randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+)from deepteam.attacks.multi_turn.sequential_break.schema import (RewrittenDialogue,DialogueJudge,@@ -27,16 +29,21 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusalfrom deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability+from deepteam.attacks.single_turn.base_single_turn_attack import (+ BaseSingleTurnAttack,+)++class SequentialJailbreak(BaseMultiTurnAttack):+ name = "Sequential Jailbreak"-class SequentialJailbreak(BaseAttack):def __init__(self,weight: int = 1,type: Optional[SequentialJailbreakTypeLiteral] = None,persona: Optional[DialogueTypeLiteral] = None,num_turns: int = 5,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weight@@ -431,7 +438,7 @@ class SequentialJailbreak(BaseAttack):def progress(self,- vulnerability: "BaseVulnerability",+ vulnerability: BaseVulnerability,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,) -> Dict[VulnerabilityType, List[RTTurn]]:@@ -514,7 +521,7 @@ class SequentialJailbreak(BaseAttack):async def a_progress(self,- vulnerability: "BaseVulnerability",+ vulnerability: BaseVulnerability,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,) -> Dict[VulnerabilityType, List[RTTurn]]:@@ -595,4 +602,4 @@ class SequentialJailbreak(BaseAttack):return resultdef get_name(self) -> str:- return "Sequential Jailbreak"+ return self.namediff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.pyindex e9aac840..f12266b7 100644--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py@@ -9,7 +9,9 @@ import randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+)from deepteam.attacks.multi_turn.tree_jailbreaking.schema import (ImprovementPrompt,Rating,@@ -27,6 +29,9 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusalfrom deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability+from deepteam.attacks.single_turn.base_single_turn_attack import (+ BaseSingleTurnAttack,+)class TreeNode:@@ -37,7 +42,7 @@ class TreeNode:depth: int,conversation_history=None,parent: Optional["TreeNode"] = None,- turn_level_attack: Optional[BaseAttack] = None,+ turn_level_attack: Optional[BaseSingleTurnAttack] = None,):self.prompt = promptself.score = score@@ -48,13 +53,14 @@ class TreeNode:self.conversation_history = conversation_history or []-class TreeJailbreaking(BaseAttack):+class TreeJailbreaking(BaseMultiTurnAttack):+ name = "Tree Jailbreaking"def __init__(self,weight: int = 1,max_depth: int = 5,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weight@@ -697,4 +703,4 @@ class TreeJailbreaking(BaseAttack):return max(MIN_BRANCHES, min(MAX_BRANCHES, branches))def get_name(self) -> str:- return "Tree Jailbreaking"+ return self.namediff --git a/deepteam/attacks/single_turn/__init__.py b/deepteam/attacks/single_turn/__init__.pyindex a5b5e80e..7268cb06 100644--- a/deepteam/attacks/single_turn/__init__.py+++ b/deepteam/attacks/single_turn/__init__.py@@ -35,26 +35,3 @@ __all__ = ["ContextPoisoning","GoalRedirection",]--SINGLE_TURN_ATTACK_MAP = {- v.__name__: v- for v in [- Base64,- GrayBox,- Leetspeak,- MathProblem,- Multilingual,- PromptInjection,- PromptProbing,- Roleplay,- ROT13,- SystemOverride,- PermissionEscalation,- LinguisticConfusion,- InputBypass,- ContextPoisoning,- GoalRedirection,- ]-}--SINGLE_TURN_ATTACK_NAMES = sorted(list(SINGLE_TURN_ATTACK_MAP.keys()))diff --git a/deepteam/attacks/single_turn/base64/base64.py b/deepteam/attacks/single_turn/base64/base64.pyindex a0d5ec89..07d88d6c 100644--- a/deepteam/attacks/single_turn/base64/base64.py+++ b/deepteam/attacks/single_turn/base64/base64.py@@ -3,6 +3,8 @@ from deepteam.attacks.single_turn import BaseSingleTurnAttackclass Base64(BaseSingleTurnAttack):+ name = "Base64"+def __init__(self, weight: int = 1):self.weight = weight@@ -11,4 +13,4 @@ class Base64(BaseSingleTurnAttack):return base64.b64encode(attack.encode()).decode()def get_name(self) -> str:- return "Base64"+ return self.namediff --git a/deepteam/attacks/single_turn/base_single_turn_attack.py b/deepteam/attacks/single_turn/base_single_turn_attack.pyindex 4e1df8bf..97a7d586 100644--- a/deepteam/attacks/single_turn/base_single_turn_attack.py+++ b/deepteam/attacks/single_turn/base_single_turn_attack.py@@ -2,4 +2,9 @@ from deepteam.attacks.base_attack import BaseAttackclass BaseSingleTurnAttack(BaseAttack):- pass++ def enhance(self, attack: str, *args, **kwargs) -> str:+ pass++ async def a_enhance(self, attack: str, *args, **kwargs) -> str:+ return self.enhance(attack, *args, **kwargs)diff --git a/deepteam/attacks/single_turn/constants.py b/deepteam/attacks/single_turn/constants.pynew file mode 100644index 00000000..b7605316--- /dev/null+++ b/deepteam/attacks/single_turn/constants.py@@ -0,0 +1,47 @@+from typing import Dict++from .base_single_turn_attack import BaseSingleTurnAttack+from .base64 import Base64+from .gray_box import GrayBox+from .leetspeak import Leetspeak+from .math_problem import MathProblem+from .multilingual import Multilingual+from .prompt_injection import PromptInjection+from .prompt_probing import PromptProbing+from .roleplay import Roleplay+from .rot13 import ROT13+from .system_override import SystemOverride+++from .system_override import SystemOverride+from .permission_escalation.permission_escalation import PermissionEscalation+from .goal_redirection.goal_redirection import GoalRedirection+from .semantic_manipulation.semantic_manipulation import LinguisticConfusion+from .input_bypass.input_bypass import InputBypass+from .context_poisoning.context_poisoning import ContextPoisoning+++SINGLE_TURN_ATTACK_CLASSES_MAP: Dict[str, BaseSingleTurnAttack] = {+ v.name: v+ for v in [+ Base64,+ GrayBox,+ Leetspeak,+ MathProblem,+ Multilingual,+ PromptInjection,+ PromptProbing,+ Roleplay,+ ROT13,+ SystemOverride,+ PermissionEscalation,+ LinguisticConfusion,+ InputBypass,+ ContextPoisoning,+ GoalRedirection,+ ]+}++SINGLE_TURN_ATTACK_NAMES = sorted(+ v.name for v in SINGLE_TURN_ATTACK_CLASSES_MAP.values()+)diff --git a/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py b/deepteam/attacks/single_turn/context_poisoning/context_poisoning.pyindex 58f90322..51886342 100644--- a/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py+++ b/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.context_poisoning.template import (class ContextPoisoning(BaseSingleTurnAttack):+ name = "Context Poisoning"+def __init__(self, weight: int = 3, max_retries: int = 3):self.weight = weight # Higher weight since memory poisoning can have persistent effectsself.max_retries = max_retriesdiff --git a/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py b/deepteam/attacks/single_turn/goal_redirection/goal_redirection.pyindex 8e0d5b80..b8da22e8 100644--- a/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py+++ b/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.goal_redirection.template import (class GoalRedirection(BaseSingleTurnAttack):+ name = "Goal Redirection"+def __init__(self, weight: int = 1, max_retries: int = 3):self.weight = weightself.max_retries = max_retries@@ -48,4 +50,4 @@ class GoalRedirection(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "Goal Redirection"+ return self.namediff --git a/deepteam/attacks/single_turn/gray_box/gray_box.py b/deepteam/attacks/single_turn/gray_box/gray_box.pyindex a0281f6c..a42c7d2b 100644--- a/deepteam/attacks/single_turn/gray_box/gray_box.py+++ b/deepteam/attacks/single_turn/gray_box/gray_box.py@@ -20,6 +20,7 @@ from deepteam.attacks.attack_simulator.utils import (class GrayBox(BaseSingleTurnAttack):+ name = "Gray Box"def __init__(self, weight: int = 1, max_retries: int = 5):self.weight = weight@@ -136,4 +137,4 @@ class GrayBox(BaseSingleTurnAttack):return attackdef get_name(self) -> str:- return "Gray Box"+ return self.namediff --git a/deepteam/attacks/single_turn/input_bypass/input_bypass.py b/deepteam/attacks/single_turn/input_bypass/input_bypass.pyindex 1a026942..0c338567 100644--- a/deepteam/attacks/single_turn/input_bypass/input_bypass.py+++ b/deepteam/attacks/single_turn/input_bypass/input_bypass.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.input_bypass.template import (class InputBypass(BaseSingleTurnAttack):+ name = "Input Bypass"+def __init__(self, weight: int = 2, max_retries: int = 3):self.weight = weight # Higher weight since this targets validation vulnerabilitiesself.max_retries = max_retries@@ -44,4 +46,4 @@ class InputBypass(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "Input Bypass"+ return self.namediff --git a/deepteam/attacks/single_turn/leetspeak/leetspeak.py b/deepteam/attacks/single_turn/leetspeak/leetspeak.pyindex 32164825..0db3c911 100644--- a/deepteam/attacks/single_turn/leetspeak/leetspeak.py+++ b/deepteam/attacks/single_turn/leetspeak/leetspeak.py@@ -2,6 +2,8 @@ from deepteam.attacks.single_turn import BaseSingleTurnAttackclass Leetspeak(BaseSingleTurnAttack):+ name = "Leetspeak"+def __init__(self, weight: int = 1):self.weight = weightdiff --git a/deepteam/attacks/single_turn/math_problem/math_problem.py b/deepteam/attacks/single_turn/math_problem/math_problem.pyindex 2f882976..2fc6309c 100644--- a/deepteam/attacks/single_turn/math_problem/math_problem.py+++ b/deepteam/attacks/single_turn/math_problem/math_problem.py@@ -21,6 +21,8 @@ from deepteam.attacks.attack_simulator.utils import (class MathProblem(BaseSingleTurnAttack):+ name = "Math Problem"+def __init__(self,weight: int = 1,@@ -147,4 +149,4 @@ class MathProblem(BaseSingleTurnAttack):)def get_name(self) -> str:- return "Math Problem"+ return self.namediff --git a/deepteam/attacks/single_turn/multilingual/multilingual.py b/deepteam/attacks/single_turn/multilingual/multilingual.pyindex 62598b8b..20ba78b0 100644--- a/deepteam/attacks/single_turn/multilingual/multilingual.py+++ b/deepteam/attacks/single_turn/multilingual/multilingual.py@@ -22,6 +22,8 @@ from deepteam.attacks.attack_simulator.utils import (class Multilingual(BaseSingleTurnAttack):+ name = "Multilingual"+def __init__(self, weight: int = 1, max_retries: int = 5):self.weight = weightself.max_retries = max_retries@@ -135,4 +137,4 @@ class Multilingual(BaseSingleTurnAttack):return attackdef get_name(self) -> str:- return "Multilingual"+ return self.namediff --git a/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py b/deepteam/attacks/single_turn/permission_escalation/permission_escalation.pyindex f27a6ffe..db466183 100644--- a/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py+++ b/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.permission_escalation.template import (class PermissionEscalation(BaseSingleTurnAttack):+ name = "Permission Escalation"+def __init__(self, weight: int = 1, max_retries: int = 3):self.weight = weightself.max_retries = max_retries@@ -48,4 +50,4 @@ class PermissionEscalation(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "Permission Escalation"+ return self.namediff --git a/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py b/deepteam/attacks/single_turn/prompt_injection/prompt_injection.pyindex b25aa73e..6018de15 100644--- a/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py+++ b/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.prompt_injection.template import (class PromptInjection(BaseSingleTurnAttack):+ name = "Prompt Injection"+def __init__(self, weight: int = 1):self.weight = weight@@ -22,4 +24,4 @@ class PromptInjection(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "Prompt Injection"+ return self.namediff --git a/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py b/deepteam/attacks/single_turn/prompt_probing/prompt_probing.pyindex f51a9ed9..1ef44768 100644--- a/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py+++ b/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py@@ -21,6 +21,7 @@ from deepteam.attacks.attack_simulator.utils import (class PromptProbing(BaseSingleTurnAttack):+ name = "Prompt Probing"def __init__(self, weight: int = 1, max_retries: int = 3):self.weight = weight@@ -140,4 +141,4 @@ class PromptProbing(BaseSingleTurnAttack):return attackdef get_name(self) -> str:- return "Prompt Probing"+ return self.namediff --git a/deepteam/attacks/single_turn/roleplay/roleplay.py b/deepteam/attacks/single_turn/roleplay/roleplay.pyindex e065c060..55a7e673 100644--- a/deepteam/attacks/single_turn/roleplay/roleplay.py+++ b/deepteam/attacks/single_turn/roleplay/roleplay.py@@ -22,6 +22,7 @@ from deepteam.attacks.attack_simulator.utils import (class Roleplay(BaseSingleTurnAttack):+ name = "Roleplay"def __init__(self,diff --git a/deepteam/attacks/single_turn/rot13/rot13.py b/deepteam/attacks/single_turn/rot13/rot13.pyindex 85c0077d..0d8af76d 100644--- a/deepteam/attacks/single_turn/rot13/rot13.py+++ b/deepteam/attacks/single_turn/rot13/rot13.py@@ -2,6 +2,8 @@ from deepteam.attacks.single_turn import BaseSingleTurnAttackclass ROT13(BaseSingleTurnAttack):+ name = "ROT-13"+def __init__(self, weight: int = 1):self.weight = weight@@ -18,4 +20,4 @@ class ROT13(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "ROT-13"+ return self.namediff --git a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.pyindex 034c1460..c24c3f27 100644--- a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py+++ b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.semantic_manipulation.template import (class LinguisticConfusion(BaseSingleTurnAttack):+ name = "Linguistic Confusion"+def __init__(self, weight: int = 1, max_retries: int = 3):self.weight = weightself.max_retries = max_retries@@ -43,4 +45,4 @@ class LinguisticConfusion(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "Linguistic Confusion"+ return self.namediff --git a/deepteam/attacks/single_turn/system_override/system_override.py b/deepteam/attacks/single_turn/system_override/system_override.pyindex bd4f2709..62426f8a 100644--- a/deepteam/attacks/single_turn/system_override/system_override.py+++ b/deepteam/attacks/single_turn/system_override/system_override.py@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.system_override.template import (class SystemOverride(BaseSingleTurnAttack):+ name = "System Override"+def __init__(self, weight: int = 1, max_retries: int = 3):self.weight = weightself.max_retries = max_retries@@ -48,4 +50,4 @@ class SystemOverride(BaseSingleTurnAttack):return self.enhance(attack)def get_name(self) -> str:- return "System Override"+ return self.namediff --git a/deepteam/red_teamer/api.py b/deepteam/red_teamer/api.pyindex 4df12421..fa6d2fbe 100644--- a/deepteam/red_teamer/api.py+++ b/deepteam/red_teamer/api.py@@ -28,9 +28,7 @@ class AttackMethodResult(BaseModel):class APIRTTurn(TurnApi):- turn_level_attack: Optional[str] = Field(- alias="turnLevelAttack"- )+ turn_level_attack: Optional[str] = Field(alias="turnLevelAttack")class APIRTTestCase(BaseModel):@@ -39,9 +37,7 @@ class APIRTTestCase(BaseModel):retrieval_context: Optional[List[str]] = Field(None, alias="retrievalContext")- tools_called: Optional[List[str]] = Field(- None, alias="toolsCalled"- )+ tools_called: Optional[List[str]] = Field(None, alias="toolsCalled")turns: Optional[List[APIRTTurn]] = Field(None)success: Optional[bool] = Field(None)@@ -61,14 +57,10 @@ class APIRiskAssessment(BaseModel):vulnerability_results: List[VulnerabilityTypeResult] = Field(alias="vulnerabilityResults")- attack_results: List[AttackMethodResult] = Field(- alias="attackResults"- )+ attack_results: List[AttackMethodResult] = Field(alias="attackResults")run_duration: float = Field(alias="runDuration")identifier: Optional[str] = Field(alias="identifier")- assessment_cost: Optional[float] = Field(- alias="assessmentCost"- )+ assessment_cost: Optional[float] = Field(alias="assessmentCost")test_cases: List[APIRTTestCase] = Field(alias="testCases")@@ -94,7 +86,6 @@ def map_test_case_to_api(test_case: RTTestCase, index: int) -> APIRTTestCase:for idx, turn in enumerate(test_case.turns)]-if test_case.score is not None:success = test_case.score > 0else:diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex b9d6deb8..7d90dad4 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -701,7 +701,6 @@ class RedTeamer:# Pydantic version below 2.0body = api_risk_assessment.dict(by_alias=True, exclude_none=True)-data, link = api.send_request(method=HttpMethods.POST,endpoint=Endpoints.RISK_ASSESSMENT_ENDPOINT,diff --git a/deepteam/vulnerabilities/__init__.py b/deepteam/vulnerabilities/__init__.pyindex 5fe88254..a60ba22f 100644--- a/deepteam/vulnerabilities/__init__.py+++ b/deepteam/vulnerabilities/__init__.py@@ -1,3 +1,4 @@+from typing import Dict, Listfrom .base_vulnerability import BaseVulnerabilityfrom .bias.bias import Biasfrom .toxicity.toxicity import Toxicity@@ -17,13 +18,10 @@ from .competition.competition import Competitionfrom .graphic_content.graphic_content import GraphicContentfrom .personal_safety.personal_safety import PersonalSafetyfrom .custom.custom import CustomVulnerability--# Agentic vulnerabilities-from .agentic.goal_theft.goal_theft import GoalTheft-from .agentic.recursive_hijacking.recursive_hijacking import RecursiveHijacking-from .agentic.robustness.robustness import Robustness-from .agentic.excessive_agency.excessive_agency import ExcessiveAgency-+from .goal_theft.goal_theft import GoalTheft+from .recursive_hijacking.recursive_hijacking import RecursiveHijacking+from .robustness.robustness import Robustness+from .excessive_agency.excessive_agency import ExcessiveAgency__all__ = ["BaseVulnerability",@@ -50,33 +48,3 @@ __all__ = ["Robustness","ExcessiveAgency",]--VULNERABILITY_MAP = {- v.__name__: v- for v in [- Bias,- Toxicity,- Misinformation,- IllegalActivity,- PromptLeakage,- PIILeakage,- BFLA,- BOLA,- RBAC,- DebugAccess,- ShellInjection,- SQLInjection,- SSRF,- IntellectualProperty,- Competition,- GraphicContent,- PersonalSafety,- CustomVulnerability,- GoalTheft,- RecursiveHijacking,- Robustness,- ExcessiveAgency,- ]-}--VULNERABILITY_NAMES = sorted(list(VULNERABILITY_MAP.keys()))diff --git a/deepteam/vulnerabilities/agentic/__init__.py b/deepteam/vulnerabilities/agentic/__init__.pydeleted file mode 100644index 10b1cab0..00000000--- a/deepteam/vulnerabilities/agentic/__init__.py+++ /dev/null@@ -1,2 +0,0 @@-from .goal_theft.goal_theft import GoalTheft-from .recursive_hijacking.recursive_hijacking import RecursiveHijackingdiff --git a/deepteam/vulnerabilities/base_vulnerability.py b/deepteam/vulnerabilities/base_vulnerability.pyindex 5df0709d..31b98445 100644--- a/deepteam/vulnerabilities/base_vulnerability.py+++ b/deepteam/vulnerabilities/base_vulnerability.py@@ -6,6 +6,8 @@ from deepteam.metrics import BaseRedTeamingMetricclass BaseVulnerability(ABC):metric: BaseRedTeamingMetric+ types: List[Enum]+ name: str = None # Subclasses should override thisdef __init__(self, types: List[Enum]):"""@@ -48,7 +50,7 @@ class BaseVulnerability(ABC):passdef get_name(self) -> str:- return self.__class__.__name__+ return self.name if self.name else self.__class__.__name__def __repr__(self):"""diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex 8f1c1c32..d8448262 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -25,6 +25,8 @@ BFLALiteral = Literal[class BFLA(BaseVulnerability):+ name = "BFLA"+def __init__(self,async_mode: bool = True,@@ -37,7 +39,7 @@ class BFLA(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=BFLAType+ self.name, types=types, allowed_type=BFLAType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -292,4 +294,4 @@ class BFLA(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "BFLA"+ return self.namediff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex da34f9aa..612532d8 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -20,6 +20,8 @@ BiasLiteralType = Literal["religion", "politics", "gender", "race"]class Bias(BaseVulnerability):+ name = "Bias"+def __init__(self,async_mode: bool = True,@@ -34,7 +36,7 @@ class Bias(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=BiasType+ self.name, types=types, allowed_type=BiasType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -292,4 +294,4 @@ class Bias(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Bias"+ return self.namediff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex c41d6672..7354505f 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -25,6 +25,8 @@ BOLALiteral = Literal[class BOLA(BaseVulnerability):+ name = "BOLA"+def __init__(self,async_mode: bool = True,@@ -37,7 +39,7 @@ class BOLA(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=BOLAType+ self.name, types=types, allowed_type=BOLAType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -293,4 +295,4 @@ class BOLA(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "BOLA"+ return self.namediff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex 6f911e50..cf699bd7 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -28,6 +28,8 @@ CompetitionLiteralType = Literal[class Competition(BaseVulnerability):+ name = "Competition"+def __init__(self,async_mode: bool = True,@@ -42,7 +44,7 @@ class Competition(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=CompetitionType+ self.name, types=types, allowed_type=CompetitionType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -298,4 +300,4 @@ class Competition(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Competition"+ return self.namediff --git a/deepteam/vulnerabilities/constants.py b/deepteam/vulnerabilities/constants.pynew file mode 100644index 00000000..3339d3d2--- /dev/null+++ b/deepteam/vulnerabilities/constants.py@@ -0,0 +1,101 @@+from typing import Dict, List+from .base_vulnerability import BaseVulnerability+from .bias.bias import Bias+from .toxicity.toxicity import Toxicity+from .misinformation.misinformation import Misinformation+from .illegal_activity.illegal_activity import IllegalActivity+from .prompt_leakage.prompt_leakage import PromptLeakage+from .pii_leakage.pii_leakage import PIILeakage+from .bfla.bfla import BFLA+from .bola.bola import BOLA+from .rbac.rbac import RBAC+from .debug_access.debug_access import DebugAccess+from .shell_injection.shell_injection import ShellInjection+from .sql_injection.sql_injection import SQLInjection+from .ssrf.ssrf import SSRF+from .intellectual_property.intellectual_property import IntellectualProperty+from .competition.competition import Competition+from .graphic_content.graphic_content import GraphicContent+from .personal_safety.personal_safety import PersonalSafety+from .goal_theft.goal_theft import GoalTheft+from .recursive_hijacking.recursive_hijacking import RecursiveHijacking+from .robustness.robustness import Robustness+from .excessive_agency.excessive_agency import ExcessiveAgency+++# Import types+from .bias.types import BiasType+from .toxicity.types import ToxicityType+from .misinformation.types import MisinformationType+from .illegal_activity.types import IllegalActivityType+from .prompt_leakage.types import PromptLeakageType+from .pii_leakage.types import PIILeakageType+from .bfla.types import BFLAType+from .bola.types import BOLAType+from .rbac.types import RBACType+from .debug_access.types import DebugAccessType+from .shell_injection.types import ShellInjectionType+from .sql_injection.types import SQLInjectionType+from .ssrf.types import SSRFType+from .intellectual_property.types import IntellectualPropertyType+from .competition.types import CompetitionType+from .graphic_content.types import GraphicContentType+from .personal_safety.types import PersonalSafetyType+from .goal_theft.types import GoalTheftType+from .recursive_hijacking.types import RecursiveHijackingType+from .robustness.types import RobustnessType+from .excessive_agency.types import ExcessiveAgencyType+++VULNERABILITY_CLASSES_MAP: Dict[str, BaseVulnerability] = {+ v.name: v+ for v in [+ Bias,+ Toxicity,+ Misinformation,+ IllegalActivity,+ PromptLeakage,+ PIILeakage,+ BFLA,+ BOLA,+ RBAC,+ DebugAccess,+ ShellInjection,+ SQLInjection,+ SSRF,+ IntellectualProperty,+ Competition,+ GraphicContent,+ PersonalSafety,+ GoalTheft,+ RecursiveHijacking,+ Robustness,+ ExcessiveAgency,+ ]+}++VULNERABILITY_TYPES_MAP: Dict[str, List[str]] = {+ Bias.name: [e.value for e in BiasType],+ Toxicity.name: [e.value for e in ToxicityType],+ Misinformation.name: [e.value for e in MisinformationType],+ IllegalActivity.name: [e.value for e in IllegalActivityType],+ PromptLeakage.name: [e.value for e in PromptLeakageType],+ PIILeakage.name: [e.value for e in PIILeakageType],+ BFLA.name: [e.value for e in BFLAType],+ BOLA.name: [e.value for e in BOLAType],+ RBAC.name: [e.value for e in RBACType],+ DebugAccess.name: [e.value for e in DebugAccessType],+ ShellInjection.name: [e.value for e in ShellInjectionType],+ SQLInjection.name: [e.value for e in SQLInjectionType],+ SSRF.name: [e.value for e in SSRFType],+ IntellectualProperty.name: [e.value for e in IntellectualPropertyType],+ Competition.name: [e.value for e in CompetitionType],+ GraphicContent.name: [e.value for e in GraphicContentType],+ PersonalSafety.name: [e.value for e in PersonalSafetyType],+ GoalTheft.name: [e.value for e in GoalTheftType],+ RecursiveHijacking.name: [e.value for e in RecursiveHijackingType],+ Robustness.name: [e.value for e in RobustnessType],+ ExcessiveAgency.name: [e.value for e in ExcessiveAgencyType],+}++VULNERABILITY_NAMES = sorted(v.name for v in VULNERABILITY_CLASSES_MAP.values())diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex 811c5c6a..0cc54998 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -25,6 +25,8 @@ DebugAccessLiteral = Literal[class DebugAccess(BaseVulnerability):+ name = "Debug Access"+def __init__(self,async_mode: bool = True,@@ -39,7 +41,7 @@ class DebugAccess(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=DebugAccessType+ self.name, types=types, allowed_type=DebugAccessType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -292,4 +294,4 @@ class DebugAccess(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Debug Access"+ return self.namediff --git a/deepteam/vulnerabilities/agentic/excessive_agency/__init__.py b/deepteam/vulnerabilities/excessive_agency/__init__.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/excessive_agency/__init__.pyrename to deepteam/vulnerabilities/excessive_agency/__init__.pydiff --git a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.pysimilarity index 98%rename from deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.pyrename to deepteam/vulnerabilities/excessive_agency/excessive_agency.pyindex edd747a4..eba5481e 100644--- a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loopfrom deepteam.utils import validate_model_callback_signaturefrom deepteam.vulnerabilities import BaseVulnerability-from deepteam.vulnerabilities.agentic.excessive_agency import (+from deepteam.vulnerabilities.excessive_agency import (ExcessiveAgencyType,)from deepteam.test_case import RTTestCase@@ -23,6 +23,8 @@ ExcessiveAgencyLiteral = Literal["functionality", "permissions", "autonomy"]class ExcessiveAgency(BaseVulnerability):+ name = "Excessive Agency"+def __init__(self,async_mode: bool = True,@@ -37,7 +39,7 @@ class ExcessiveAgency(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=ExcessiveAgencyType+ self.name, types=types, allowed_type=ExcessiveAgencyType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -292,4 +294,4 @@ class ExcessiveAgency(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Excessive Agency"+ return self.namediff --git a/deepteam/vulnerabilities/agentic/excessive_agency/template.py b/deepteam/vulnerabilities/excessive_agency/template.pysimilarity index 99%rename from deepteam/vulnerabilities/agentic/excessive_agency/template.pyrename to deepteam/vulnerabilities/excessive_agency/template.pyindex de246d1e..c2a2b7f8 100644--- a/deepteam/vulnerabilities/agentic/excessive_agency/template.py+++ b/deepteam/vulnerabilities/excessive_agency/template.py@@ -1,6 +1,6 @@from typing import Optional-from deepteam.vulnerabilities.agentic.excessive_agency.types import (+from deepteam.vulnerabilities.excessive_agency.types import (ExcessiveAgencyType,)diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/types.py b/deepteam/vulnerabilities/excessive_agency/types.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/excessive_agency/types.pyrename to deepteam/vulnerabilities/excessive_agency/types.pydiff --git a/deepteam/vulnerabilities/agentic/goal_theft/__init__.py b/deepteam/vulnerabilities/goal_theft/__init__.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/goal_theft/__init__.pyrename to deepteam/vulnerabilities/goal_theft/__init__.pydiff --git a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.pysimilarity index 98%rename from deepteam/vulnerabilities/agentic/goal_theft/goal_theft.pyrename to deepteam/vulnerabilities/goal_theft/goal_theft.pyindex 18f04889..2582b178 100644--- a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loopfrom deepteam.utils import validate_model_callback_signaturefrom deepteam.vulnerabilities import BaseVulnerability-from deepteam.vulnerabilities.agentic.goal_theft import GoalTheftType+from deepteam.vulnerabilities.goal_theft import GoalTheftTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics.agentic.extraction_success.extraction_success import (ExtractionSuccessMetric,@@ -25,6 +25,8 @@ GoalTheftLiteralType = Literal[class GoalTheft(BaseVulnerability):+ name = "Goal Theft"+def __init__(self,async_mode: bool = True,@@ -39,7 +41,7 @@ class GoalTheft(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=GoalTheftType+ self.name, types=types, allowed_type=GoalTheftType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -294,4 +296,4 @@ class GoalTheft(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Goal Theft"+ return self.namediff --git a/deepteam/vulnerabilities/agentic/goal_theft/template.py b/deepteam/vulnerabilities/goal_theft/template.pysimilarity index 99%rename from deepteam/vulnerabilities/agentic/goal_theft/template.pyrename to deepteam/vulnerabilities/goal_theft/template.pyindex b46a98ef..e3cdf4b9 100644--- a/deepteam/vulnerabilities/agentic/goal_theft/template.py+++ b/deepteam/vulnerabilities/goal_theft/template.py@@ -1,6 +1,6 @@from typing import Optional-from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType+from deepteam.vulnerabilities.goal_theft.types import GoalTheftTypeclass GoalTheftTemplate:diff --git a/deepteam/vulnerabilities/agentic/goal_theft/types.py b/deepteam/vulnerabilities/goal_theft/types.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/goal_theft/types.pyrename to deepteam/vulnerabilities/goal_theft/types.pydiff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 24ef9ae1..9ee5a3aa 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -23,6 +23,8 @@ GraphicContentLiteral = Literal[class GraphicContent(BaseVulnerability):+ name = "Graphic Content"+def __init__(self,async_mode: bool = True,@@ -37,7 +39,7 @@ class GraphicContent(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=GraphicContentType+ self.name, types=types, allowed_type=GraphicContentType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -294,4 +296,4 @@ class GraphicContent(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Graphic Content"+ return self.namediff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex 003ec6b3..e28cc246 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -29,6 +29,8 @@ IllegalActivityLiteral = Literal[class IllegalActivity(BaseVulnerability):+ name = "Illegal Activity"+def __init__(self,async_mode: bool = True,@@ -43,7 +45,7 @@ class IllegalActivity(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=IllegalActivityType+ self.name, types=types, allowed_type=IllegalActivityType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -301,4 +303,4 @@ class IllegalActivity(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Illegal Activity"+ return self.namediff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex 258b9a5c..6f34c29f 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -28,6 +28,8 @@ IntellectualPropertyLiteral = Literal[class IntellectualProperty(BaseVulnerability):+ name = "Intellectual Property"+def __init__(self,async_mode: bool = True,@@ -42,7 +44,7 @@ class IntellectualProperty(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=IntellectualPropertyType+ self.name, types=types, allowed_type=IntellectualPropertyType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -299,4 +301,4 @@ class IntellectualProperty(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Intellectual Property"+ return self.namediff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex df57acbc..12698444 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -24,6 +24,8 @@ MisinformationLiteral = Literal[class Misinformation(BaseVulnerability):+ name = "Misinformation"+def __init__(self,async_mode: bool = True,@@ -38,7 +40,7 @@ class Misinformation(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=MisinformationType+ self.name, types=types, allowed_type=MisinformationType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -294,4 +296,4 @@ class Misinformation(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Misinformation"+ return self.namediff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex 3bbb4e89..8be4d671 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -27,6 +27,8 @@ PersonalSafetyLiteral = Literal[class PersonalSafety(BaseVulnerability):+ name = "Personal Safety"+def __init__(self,async_mode: bool = True,@@ -41,7 +43,7 @@ class PersonalSafety(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=PersonalSafetyType+ self.name, types=types, allowed_type=PersonalSafetyType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -293,4 +295,4 @@ class PersonalSafety(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Personal Safety"+ return self.namediff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex e0980623..2d7d61e6 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -26,6 +26,8 @@ PIILeakageLiteral = Literal[class PIILeakage(BaseVulnerability):+ name = "PII Leakage"+def __init__(self,async_mode: bool = True,@@ -40,7 +42,7 @@ class PIILeakage(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=PIILeakageType+ self.name, types=types, allowed_type=PIILeakageType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -300,4 +302,4 @@ class PIILeakage(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "PII Leakage"+ return self.namediff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex 2d81deef..8bf44b2e 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -26,6 +26,8 @@ PromptLeakageLiteral = Literal[class PromptLeakage(BaseVulnerability):+ name = "Prompt Leakage"+def __init__(self,async_mode: bool = True,@@ -40,7 +42,7 @@ class PromptLeakage(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=PromptLeakageType+ self.name, types=types, allowed_type=PromptLeakageType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -297,4 +299,4 @@ class PromptLeakage(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Prompt Leakage"+ return self.namediff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex f13c73e1..233983de 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -25,6 +25,8 @@ RBACLiteral = Literal[class RBAC(BaseVulnerability):+ name = "RBAC"+def __init__(self,async_mode: bool = True,@@ -37,7 +39,7 @@ class RBAC(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=RBACType+ self.name, types=types, allowed_type=RBACType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -294,4 +296,4 @@ class RBAC(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "RBAC"+ return self.namediff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/__init__.py b/deepteam/vulnerabilities/recursive_hijacking/__init__.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/recursive_hijacking/__init__.pyrename to deepteam/vulnerabilities/recursive_hijacking/__init__.pydiff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.pysimilarity index 98%rename from deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.pyrename to deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.pyindex b04c73ea..000db93a 100644--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loopfrom deepteam.utils import validate_model_callback_signaturefrom deepteam.vulnerabilities import BaseVulnerability-from deepteam.vulnerabilities.agentic.recursive_hijacking import (+from deepteam.vulnerabilities.recursive_hijacking import (RecursiveHijackingType,)from deepteam.vulnerabilities.utils import validate_vulnerability_types@@ -29,6 +29,8 @@ RecursiveHijackingLiteralType = Literal[class RecursiveHijacking(BaseVulnerability):+ name = "Recursive Hijacking"+def __init__(self,async_mode: bool = True,@@ -43,7 +45,7 @@ class RecursiveHijacking(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=RecursiveHijackingType+ self.name, types=types, allowed_type=RecursiveHijackingType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -298,4 +300,4 @@ class RecursiveHijacking(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Recursive Hijacking"+ return self.namediff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/template.py b/deepteam/vulnerabilities/recursive_hijacking/template.pysimilarity index 99%rename from deepteam/vulnerabilities/agentic/recursive_hijacking/template.pyrename to deepteam/vulnerabilities/recursive_hijacking/template.pyindex 76bc3a1a..2b7b93cf 100644--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/template.py+++ b/deepteam/vulnerabilities/recursive_hijacking/template.py@@ -1,6 +1,6 @@from typing import Optional-from deepteam.vulnerabilities.agentic.recursive_hijacking.types import (+from deepteam.vulnerabilities.recursive_hijacking.types import (RecursiveHijackingType,)diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/types.py b/deepteam/vulnerabilities/recursive_hijacking/types.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/recursive_hijacking/types.pyrename to deepteam/vulnerabilities/recursive_hijacking/types.pydiff --git a/deepteam/vulnerabilities/agentic/robustness/__init__.py b/deepteam/vulnerabilities/robustness/__init__.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/robustness/__init__.pyrename to deepteam/vulnerabilities/robustness/__init__.pydiff --git a/deepteam/vulnerabilities/agentic/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.pysimilarity index 98%rename from deepteam/vulnerabilities/agentic/robustness/robustness.pyrename to deepteam/vulnerabilities/robustness/robustness.pyindex 3879e494..5095f923 100644--- a/deepteam/vulnerabilities/agentic/robustness/robustness.py+++ b/deepteam/vulnerabilities/robustness/robustness.py@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loopfrom deepteam.utils import validate_model_callback_signaturefrom deepteam.vulnerabilities import BaseVulnerability-from deepteam.vulnerabilities.agentic.robustness import RobustnessType+from deepteam.vulnerabilities.robustness import RobustnessTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import (HijackingMetric,@@ -24,6 +24,8 @@ RobustnessLiteral = Literal["input_overreliance", "hijacking"]class Robustness(BaseVulnerability):+ name = "Robustness"+def __init__(self,async_mode: bool = True,@@ -38,7 +40,7 @@ class Robustness(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=RobustnessType+ self.name, types=types, allowed_type=RobustnessType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -305,4 +307,4 @@ class Robustness(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Robustness"+ return self.namediff --git a/deepteam/vulnerabilities/agentic/robustness/template.py b/deepteam/vulnerabilities/robustness/template.pysimilarity index 99%rename from deepteam/vulnerabilities/agentic/robustness/template.pyrename to deepteam/vulnerabilities/robustness/template.pyindex b6d72ff6..a0bdf1a7 100644--- a/deepteam/vulnerabilities/agentic/robustness/template.py+++ b/deepteam/vulnerabilities/robustness/template.py@@ -1,6 +1,6 @@from typing import Optional-from deepteam.vulnerabilities.agentic.robustness.types import RobustnessType+from deepteam.vulnerabilities.robustness.types import RobustnessTypeclass RobustnessTemplate:diff --git a/deepteam/vulnerabilities/agentic/robustness/types.py b/deepteam/vulnerabilities/robustness/types.pysimilarity index 100%rename from deepteam/vulnerabilities/agentic/robustness/types.pyrename to deepteam/vulnerabilities/robustness/types.pydiff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex 35413b16..3e1386b9 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -25,6 +25,8 @@ ShellInjectionLiteral = Literal[class ShellInjection(BaseVulnerability):+ name = "Shell Injection"+def __init__(self,async_mode: bool = True,@@ -39,7 +41,7 @@ class ShellInjection(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=ShellInjectionType+ self.name, types=types, allowed_type=ShellInjectionType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -289,4 +291,4 @@ class ShellInjection(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Shell Injection"+ return self.namediff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex 705c3de5..e888f55e 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -25,6 +25,8 @@ SQLInjectionLiteral = Literal[class SQLInjection(BaseVulnerability):+ name = "SQL Injection"+def __init__(self,async_mode: bool = True,@@ -39,7 +41,7 @@ class SQLInjection(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=SQLInjectionType+ self.name, types=types, allowed_type=SQLInjectionType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -289,4 +291,4 @@ class SQLInjection(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "SQL Injection"+ return self.namediff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex df2e279b..e80730ac 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -25,6 +25,8 @@ SSRFLiteral = Literal[class SSRF(BaseVulnerability):+ name = "SSRF"+def __init__(self,async_mode: bool = True,@@ -37,7 +39,7 @@ class SSRF(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=SSRFType+ self.name, types=types, allowed_type=SSRFType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -288,4 +290,4 @@ class SSRF(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "SSRF"+ return self.namediff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex 36bfde3d..ef5464bf 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -21,6 +21,8 @@ ToxicityLiteral = Literal["profanity", "insults", "threats", "mockery"]class Toxicity(BaseVulnerability):+ name = "Toxicity"+def __init__(self,async_mode: bool = True,@@ -35,7 +37,7 @@ class Toxicity(BaseVulnerability):purpose: Optional[str] = None,):enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=ToxicityType+ self.name, types=types, allowed_type=ToxicityType)self.async_mode = async_modeself.verbose_mode = verbose_mode@@ -289,4 +291,4 @@ class Toxicity(BaseVulnerability):return self.vulnerabledef get_name(self) -> str:- return "Toxicity"+ return self.namediff --git a/deepteam/vulnerabilities/types.py b/deepteam/vulnerabilities/types.pyindex bd54ccfd..94b04de8 100644--- a/deepteam/vulnerabilities/types.py+++ b/deepteam/vulnerabilities/types.py@@ -50,26 +50,26 @@ from deepteam.vulnerabilities.shell_injection.template import (ShellInjectionTemplate,)from deepteam.vulnerabilities.sql_injection.template import SQLInjectionTemplate-from deepteam.vulnerabilities.agentic.robustness import RobustnessType-from deepteam.vulnerabilities.agentic.robustness.template import (+from deepteam.vulnerabilities.robustness import RobustnessType+from deepteam.vulnerabilities.robustness.template import (RobustnessTemplate,)-from deepteam.vulnerabilities.agentic.excessive_agency import (+from deepteam.vulnerabilities.excessive_agency import (ExcessiveAgencyType,)-from deepteam.vulnerabilities.agentic.excessive_agency.template import (+from deepteam.vulnerabilities.excessive_agency.template import (ExcessiveAgencyTemplate,)# Import agentic vulnerability types-from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType-from deepteam.vulnerabilities.agentic.recursive_hijacking.types import (+from deepteam.vulnerabilities.goal_theft.types import GoalTheftType+from deepteam.vulnerabilities.recursive_hijacking.types import (RecursiveHijackingType,)-from deepteam.vulnerabilities.agentic.goal_theft.template import (+from deepteam.vulnerabilities.goal_theft.template import (GoalTheftTemplate,)-from deepteam.vulnerabilities.agentic.recursive_hijacking.template import (+from deepteam.vulnerabilities.recursive_hijacking.template import (RecursiveHijackingTemplate,)diff --git a/kk.py b/kk.pyindex a3c41c32..c62a4d97 100644--- a/kk.py+++ b/kk.py@@ -1,17 +1,39 @@from deepteam import red_team-from deepteam.vulnerabilities import Bias+from deepteam.vulnerabilities import Bias, BOLAfrom deepteam.attacks.single_turn import Roleplayimport os-async def your_callback(input: str) -> str:- return "I'm sorry but I can't answer this: " + input+# async def your_callback(input: str) -> str:+# return "I'm sorry but I can't answer this: " + input-print(os.environ.get("CONFIDENT_API_KEY"))+# print(os.environ.get("CONFIDENT_API_KEY"))-risk_assessment = red_team(- attacks=[Roleplay()],- vulnerabilities=[Bias()],- model_callback=your_callback,+# risk_assessment = red_team(+# attacks=[Roleplay()],+# vulnerabilities=[Bias(), BOLA()],+# model_callback=your_callback,+# )++from deepteam.vulnerabilities.constants import (+ VULNERABILITY_NAMES,+ VULNERABILITY_CLASSES_MAP,+ VULNERABILITY_TYPES_MAP,+)+from deepteam.attacks.single_turn.constants import (+ SINGLE_TURN_ATTACK_NAMES,+ SINGLE_TURN_ATTACK_CLASSES_MAP,)+from deepteam.attacks.multi_turn.constants import (+ MULTI_TURN_ATTACK_NAMES,+ MULTI_TURN_ATTACK_CLASSES_MAP,+)++print(SINGLE_TURN_ATTACK_NAMES, "@@@")+print(SINGLE_TURN_ATTACK_CLASSES_MAP, "@@@")+print(MULTI_TURN_ATTACK_NAMES, "@@@")+print(MULTI_TURN_ATTACK_CLASSES_MAP, "@@@")+print(VULNERABILITY_NAMES, "@@@")+print(VULNERABILITY_CLASSES_MAP, "@@@")+print(VULNERABILITY_TYPES_MAP, "@@@")