Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

.

penguine8a83cb2dc3fb
diff --git a/deepteam/attacks/base_attack.py b/deepteam/attacks/base_attack.py
index df84f43c..0ed22a7a 100644
--- a/deepteam/attacks/base_attack.py
+++ b/deepteam/attacks/base_attack.py
@@ -4,13 +4,7 @@ from abc import ABC, abstractmethod
class BaseAttack(ABC):
weight: int = 1
multi_turn: bool = False
+ name: str = None # Subclasses should override this
- def enhance(self, attack: str, *args, **kwargs) -> str:
- pass
-
- async def a_enhance(self, attack: str, *args, **kwargs) -> str:
- return self.enhance(attack, *args, **kwargs)
-
- @abstractmethod
def get_name(self) -> str:
- raise NotImplementedError
+ return self.name if self.name else self.__class__.__name__
diff --git a/deepteam/attacks/multi_turn/__init__.py b/deepteam/attacks/multi_turn/__init__.py
index d1d46a64..95163e5f 100644
--- a/deepteam/attacks/multi_turn/__init__.py
+++ b/deepteam/attacks/multi_turn/__init__.py
@@ -11,16 +11,3 @@ __all__ = [
"SequentialJailbreak",
"BadLikertJudge",
]
-
-MULTI_TURN_ATTACK_MAP = {
- v.__name__: v
- for v in [
- CrescendoJailbreaking,
- LinearJailbreaking,
- TreeJailbreaking,
- SequentialJailbreak,
- BadLikertJudge,
- ]
-}
-
-MULTI_TURN_ATTACK_NAMES = sorted(list(MULTI_TURN_ATTACK_MAP.keys()))
diff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
index 230fa30a..5480f893 100644
--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
@@ -4,7 +4,9 @@ from typing import Optional, Union, List, Dict
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
import random
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn.base_multi_turn_attack import (
+ BaseMultiTurnAttack,
+)
from deepteam.attacks.multi_turn.bad_likert_judge.schema import (
BLJJudge,
)
@@ -18,20 +20,25 @@ from deepteam.attacks.attack_simulator.utils import (
from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack
from deepteam.attacks.multi_turn.types import CallbackType
from deepteam.attacks.multi_turn.base_schema import NonRefusal
+from deepteam.attacks.single_turn.base_single_turn_attack import (
+ BaseSingleTurnAttack,
+)
from deepteam.errors import ModelRefusalError
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
-class BadLikertJudge(BaseAttack):
+class BadLikertJudge(BaseMultiTurnAttack):
+ name = "Bad Likert Judge"
+
def __init__(
self,
weight: int = 1,
category: str = "bias",
num_turns: int = 5,
enable_refinement: bool = True,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
@@ -457,7 +464,7 @@ class BadLikertJudge(BaseAttack):
return result
def get_name(self) -> str:
- return "Bad Likert Judge"
+ return self.name
@classmethod
def get_supported_categories(cls) -> list[str]:
diff --git a/deepteam/attacks/multi_turn/base_multi_turn_attack.py b/deepteam/attacks/multi_turn/base_multi_turn_attack.py
new file mode 100644
index 00000000..d19df19c
--- /dev/null
+++ b/deepteam/attacks/multi_turn/base_multi_turn_attack.py
@@ -0,0 +1,26 @@
+from deepteam.attacks.base_attack import BaseAttack
+from deepteam.vulnerabilities.base_vulnerability import BaseVulnerability
+from deepteam.attacks.multi_turn.types import CallbackType
+from typing import Optional, List, Dict
+from deepteam.test_case import RTTurn
+from deepteam.vulnerabilities.types import VulnerabilityType
+
+
+class BaseMultiTurnAttack(BaseAttack):
+ def progress(
+ self,
+ vulnerability: BaseVulnerability,
+ model_callback: CallbackType,
+ turns: Optional[List[RTTurn]] = None,
+ **kwargs,
+ ) -> Dict[VulnerabilityType, List[RTTurn]]:
+ pass
+
+ async def a_progress(
+ self,
+ vulnerability: BaseVulnerability,
+ model_callback: CallbackType,
+ turns: Optional[List[RTTurn]] = None,
+ **kwargs,
+ ) -> Dict[VulnerabilityType, List[RTTurn]]:
+ pass
diff --git a/deepteam/attacks/multi_turn/constants.py b/deepteam/attacks/multi_turn/constants.py
new file mode 100644
index 00000000..d016665d
--- /dev/null
+++ b/deepteam/attacks/multi_turn/constants.py
@@ -0,0 +1,23 @@
+from typing import Dict
+
+from .base_multi_turn_attack import BaseMultiTurnAttack
+from .crescendo_jailbreaking import CrescendoJailbreaking
+from .linear_jailbreaking import LinearJailbreaking
+from .tree_jailbreaking import TreeJailbreaking
+from .sequential_break import SequentialJailbreak
+from .bad_likert_judge import BadLikertJudge
+
+MULTI_TURN_ATTACK_CLASSES_MAP: Dict[str, BaseMultiTurnAttack] = {
+ v.name: v
+ for v in [
+ CrescendoJailbreaking,
+ LinearJailbreaking,
+ TreeJailbreaking,
+ SequentialJailbreak,
+ BadLikertJudge,
+ ]
+}
+
+MULTI_TURN_ATTACK_NAMES = sorted(
+ v.name for v in MULTI_TURN_ATTACK_CLASSES_MAP.values()
+)
diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
index 5b0a7708..27e226c8 100644
--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
@@ -9,7 +9,9 @@ from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn.base_multi_turn_attack import (
+ BaseMultiTurnAttack,
+)
from deepteam.attacks.multi_turn.crescendo_jailbreaking.template import (
JailBreakingCrescendoTemplate,
)
@@ -23,6 +25,9 @@ from deepteam.attacks.attack_simulator.utils import (
a_generate,
)
from deepteam.attacks.multi_turn.types import CallbackType
+from deepteam.attacks.single_turn.base_single_turn_attack import (
+ BaseSingleTurnAttack,
+)
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
@@ -50,14 +55,15 @@ class MemorySystem:
return new_conversation_id
-class CrescendoJailbreaking(BaseAttack):
+class CrescendoJailbreaking(BaseMultiTurnAttack):
+ name = "Crescendo Jailbreaking"
def __init__(
self,
weight: int = 1,
max_rounds: int = 10,
max_backtracks: int = 10,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
@@ -422,7 +428,7 @@ class CrescendoJailbreaking(BaseAttack):
def progress(
self,
- vulnerability: "BaseVulnerability",
+ vulnerability: BaseVulnerability,
model_callback: CallbackType,
turns: Optional[List[RTTurn]] = None,
) -> Dict[VulnerabilityType, List[RTTurn]]:
@@ -505,7 +511,7 @@ class CrescendoJailbreaking(BaseAttack):
async def a_progress(
self,
- vulnerability: "BaseVulnerability",
+ vulnerability: BaseVulnerability,
model_callback: CallbackType,
turns: Optional[List[RTTurn]] = None,
) -> Dict[VulnerabilityType, List[RTTurn]]:
@@ -778,4 +784,4 @@ class CrescendoJailbreaking(BaseAttack):
return eval_response.value, eval_response.metadata
def get_name(self) -> str:
- return "Crescendo Jailbreaking"
+ return self.name
diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
index ce01515c..f114e909 100644
--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
@@ -18,7 +18,9 @@ from deepteam.attacks.attack_simulator.utils import (
a_generate,
)
from deepteam.attacks.multi_turn.types import CallbackType
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn.base_multi_turn_attack import (
+ BaseMultiTurnAttack,
+)
from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack
from deepteam.attacks.multi_turn.base_schema import NonRefusal
from deepteam.attacks.multi_turn.base_template import BaseMultiTurnTemplate
@@ -26,14 +28,19 @@ from deepteam.errors import ModelRefusalError
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
+from deepteam.attacks.single_turn.base_single_turn_attack import (
+ BaseSingleTurnAttack,
+)
+
+class LinearJailbreaking(BaseMultiTurnAttack):
+ name = "Linear Jailbreaking"
-class LinearJailbreaking(BaseAttack):
def __init__(
self,
weight: int = 1,
num_turns: int = 5,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
@@ -430,4 +437,4 @@ class LinearJailbreaking(BaseAttack):
return result
def get_name(self) -> str:
- return "Linear Jailbreaking"
+ return self.name
diff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
index 654a0faf..aeeadd79 100644
--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
@@ -5,7 +5,9 @@ import random
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn.base_multi_turn_attack import (
+ BaseMultiTurnAttack,
+)
from deepteam.attacks.multi_turn.sequential_break.schema import (
RewrittenDialogue,
DialogueJudge,
@@ -27,16 +29,21 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusal
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
+from deepteam.attacks.single_turn.base_single_turn_attack import (
+ BaseSingleTurnAttack,
+)
+
+class SequentialJailbreak(BaseMultiTurnAttack):
+ name = "Sequential Jailbreak"
-class SequentialJailbreak(BaseAttack):
def __init__(
self,
weight: int = 1,
type: Optional[SequentialJailbreakTypeLiteral] = None,
persona: Optional[DialogueTypeLiteral] = None,
num_turns: int = 5,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
@@ -431,7 +438,7 @@ class SequentialJailbreak(BaseAttack):
def progress(
self,
- vulnerability: "BaseVulnerability",
+ vulnerability: BaseVulnerability,
model_callback: CallbackType,
turns: Optional[List[RTTurn]] = None,
) -> Dict[VulnerabilityType, List[RTTurn]]:
@@ -514,7 +521,7 @@ class SequentialJailbreak(BaseAttack):
async def a_progress(
self,
- vulnerability: "BaseVulnerability",
+ vulnerability: BaseVulnerability,
model_callback: CallbackType,
turns: Optional[List[RTTurn]] = None,
) -> Dict[VulnerabilityType, List[RTTurn]]:
@@ -595,4 +602,4 @@ class SequentialJailbreak(BaseAttack):
return result
def get_name(self) -> str:
- return "Sequential Jailbreak"
+ return self.name
diff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
index e9aac840..f12266b7 100644
--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
@@ -9,7 +9,9 @@ import random
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn.base_multi_turn_attack import (
+ BaseMultiTurnAttack,
+)
from deepteam.attacks.multi_turn.tree_jailbreaking.schema import (
ImprovementPrompt,
Rating,
@@ -27,6 +29,9 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusal
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
+from deepteam.attacks.single_turn.base_single_turn_attack import (
+ BaseSingleTurnAttack,
+)
class TreeNode:
@@ -37,7 +42,7 @@ class TreeNode:
depth: int,
conversation_history=None,
parent: Optional["TreeNode"] = None,
- turn_level_attack: Optional[BaseAttack] = None,
+ turn_level_attack: Optional[BaseSingleTurnAttack] = None,
):
self.prompt = prompt
self.score = score
@@ -48,13 +53,14 @@ class TreeNode:
self.conversation_history = conversation_history or []
-class TreeJailbreaking(BaseAttack):
+class TreeJailbreaking(BaseMultiTurnAttack):
+ name = "Tree Jailbreaking"
def __init__(
self,
weight: int = 1,
max_depth: int = 5,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
@@ -697,4 +703,4 @@ class TreeJailbreaking(BaseAttack):
return max(MIN_BRANCHES, min(MAX_BRANCHES, branches))
def get_name(self) -> str:
- return "Tree Jailbreaking"
+ return self.name
diff --git a/deepteam/attacks/single_turn/__init__.py b/deepteam/attacks/single_turn/__init__.py
index a5b5e80e..7268cb06 100644
--- a/deepteam/attacks/single_turn/__init__.py
+++ b/deepteam/attacks/single_turn/__init__.py
@@ -35,26 +35,3 @@ __all__ = [
"ContextPoisoning",
"GoalRedirection",
]
-
-SINGLE_TURN_ATTACK_MAP = {
- v.__name__: v
- for v in [
- Base64,
- GrayBox,
- Leetspeak,
- MathProblem,
- Multilingual,
- PromptInjection,
- PromptProbing,
- Roleplay,
- ROT13,
- SystemOverride,
- PermissionEscalation,
- LinguisticConfusion,
- InputBypass,
- ContextPoisoning,
- GoalRedirection,
- ]
-}
-
-SINGLE_TURN_ATTACK_NAMES = sorted(list(SINGLE_TURN_ATTACK_MAP.keys()))
diff --git a/deepteam/attacks/single_turn/base64/base64.py b/deepteam/attacks/single_turn/base64/base64.py
index a0d5ec89..07d88d6c 100644
--- a/deepteam/attacks/single_turn/base64/base64.py
+++ b/deepteam/attacks/single_turn/base64/base64.py
@@ -3,6 +3,8 @@ from deepteam.attacks.single_turn import BaseSingleTurnAttack
class Base64(BaseSingleTurnAttack):
+ name = "Base64"
+
def __init__(self, weight: int = 1):
self.weight = weight
@@ -11,4 +13,4 @@ class Base64(BaseSingleTurnAttack):
return base64.b64encode(attack.encode()).decode()
def get_name(self) -> str:
- return "Base64"
+ return self.name
diff --git a/deepteam/attacks/single_turn/base_single_turn_attack.py b/deepteam/attacks/single_turn/base_single_turn_attack.py
index 4e1df8bf..97a7d586 100644
--- a/deepteam/attacks/single_turn/base_single_turn_attack.py
+++ b/deepteam/attacks/single_turn/base_single_turn_attack.py
@@ -2,4 +2,9 @@ from deepteam.attacks.base_attack import BaseAttack
class BaseSingleTurnAttack(BaseAttack):
- pass
+
+ def enhance(self, attack: str, *args, **kwargs) -> str:
+ pass
+
+ async def a_enhance(self, attack: str, *args, **kwargs) -> str:
+ return self.enhance(attack, *args, **kwargs)
diff --git a/deepteam/attacks/single_turn/constants.py b/deepteam/attacks/single_turn/constants.py
new file mode 100644
index 00000000..b7605316
--- /dev/null
+++ b/deepteam/attacks/single_turn/constants.py
@@ -0,0 +1,47 @@
+from typing import Dict
+
+from .base_single_turn_attack import BaseSingleTurnAttack
+from .base64 import Base64
+from .gray_box import GrayBox
+from .leetspeak import Leetspeak
+from .math_problem import MathProblem
+from .multilingual import Multilingual
+from .prompt_injection import PromptInjection
+from .prompt_probing import PromptProbing
+from .roleplay import Roleplay
+from .rot13 import ROT13
+from .system_override import SystemOverride
+
+
+from .system_override import SystemOverride
+from .permission_escalation.permission_escalation import PermissionEscalation
+from .goal_redirection.goal_redirection import GoalRedirection
+from .semantic_manipulation.semantic_manipulation import LinguisticConfusion
+from .input_bypass.input_bypass import InputBypass
+from .context_poisoning.context_poisoning import ContextPoisoning
+
+
+SINGLE_TURN_ATTACK_CLASSES_MAP: Dict[str, BaseSingleTurnAttack] = {
+ v.name: v
+ for v in [
+ Base64,
+ GrayBox,
+ Leetspeak,
+ MathProblem,
+ Multilingual,
+ PromptInjection,
+ PromptProbing,
+ Roleplay,
+ ROT13,
+ SystemOverride,
+ PermissionEscalation,
+ LinguisticConfusion,
+ InputBypass,
+ ContextPoisoning,
+ GoalRedirection,
+ ]
+}
+
+SINGLE_TURN_ATTACK_NAMES = sorted(
+ v.name for v in SINGLE_TURN_ATTACK_CLASSES_MAP.values()
+)
diff --git a/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py b/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py
index 58f90322..51886342 100644
--- a/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py
+++ b/deepteam/attacks/single_turn/context_poisoning/context_poisoning.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.context_poisoning.template import (
class ContextPoisoning(BaseSingleTurnAttack):
+ name = "Context Poisoning"
+
def __init__(self, weight: int = 3, max_retries: int = 3):
self.weight = weight # Higher weight since memory poisoning can have persistent effects
self.max_retries = max_retries
diff --git a/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py b/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py
index 8e0d5b80..b8da22e8 100644
--- a/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py
+++ b/deepteam/attacks/single_turn/goal_redirection/goal_redirection.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.goal_redirection.template import (
class GoalRedirection(BaseSingleTurnAttack):
+ name = "Goal Redirection"
+
def __init__(self, weight: int = 1, max_retries: int = 3):
self.weight = weight
self.max_retries = max_retries
@@ -48,4 +50,4 @@ class GoalRedirection(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "Goal Redirection"
+ return self.name
diff --git a/deepteam/attacks/single_turn/gray_box/gray_box.py b/deepteam/attacks/single_turn/gray_box/gray_box.py
index a0281f6c..a42c7d2b 100644
--- a/deepteam/attacks/single_turn/gray_box/gray_box.py
+++ b/deepteam/attacks/single_turn/gray_box/gray_box.py
@@ -20,6 +20,7 @@ from deepteam.attacks.attack_simulator.utils import (
class GrayBox(BaseSingleTurnAttack):
+ name = "Gray Box"
def __init__(self, weight: int = 1, max_retries: int = 5):
self.weight = weight
@@ -136,4 +137,4 @@ class GrayBox(BaseSingleTurnAttack):
return attack
def get_name(self) -> str:
- return "Gray Box"
+ return self.name
diff --git a/deepteam/attacks/single_turn/input_bypass/input_bypass.py b/deepteam/attacks/single_turn/input_bypass/input_bypass.py
index 1a026942..0c338567 100644
--- a/deepteam/attacks/single_turn/input_bypass/input_bypass.py
+++ b/deepteam/attacks/single_turn/input_bypass/input_bypass.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.input_bypass.template import (
class InputBypass(BaseSingleTurnAttack):
+ name = "Input Bypass"
+
def __init__(self, weight: int = 2, max_retries: int = 3):
self.weight = weight # Higher weight since this targets validation vulnerabilities
self.max_retries = max_retries
@@ -44,4 +46,4 @@ class InputBypass(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "Input Bypass"
+ return self.name
diff --git a/deepteam/attacks/single_turn/leetspeak/leetspeak.py b/deepteam/attacks/single_turn/leetspeak/leetspeak.py
index 32164825..0db3c911 100644
--- a/deepteam/attacks/single_turn/leetspeak/leetspeak.py
+++ b/deepteam/attacks/single_turn/leetspeak/leetspeak.py
@@ -2,6 +2,8 @@ from deepteam.attacks.single_turn import BaseSingleTurnAttack
class Leetspeak(BaseSingleTurnAttack):
+ name = "Leetspeak"
+
def __init__(self, weight: int = 1):
self.weight = weight
diff --git a/deepteam/attacks/single_turn/math_problem/math_problem.py b/deepteam/attacks/single_turn/math_problem/math_problem.py
index 2f882976..2fc6309c 100644
--- a/deepteam/attacks/single_turn/math_problem/math_problem.py
+++ b/deepteam/attacks/single_turn/math_problem/math_problem.py
@@ -21,6 +21,8 @@ from deepteam.attacks.attack_simulator.utils import (
class MathProblem(BaseSingleTurnAttack):
+ name = "Math Problem"
+
def __init__(
self,
weight: int = 1,
@@ -147,4 +149,4 @@ class MathProblem(BaseSingleTurnAttack):
)
def get_name(self) -> str:
- return "Math Problem"
+ return self.name
diff --git a/deepteam/attacks/single_turn/multilingual/multilingual.py b/deepteam/attacks/single_turn/multilingual/multilingual.py
index 62598b8b..20ba78b0 100644
--- a/deepteam/attacks/single_turn/multilingual/multilingual.py
+++ b/deepteam/attacks/single_turn/multilingual/multilingual.py
@@ -22,6 +22,8 @@ from deepteam.attacks.attack_simulator.utils import (
class Multilingual(BaseSingleTurnAttack):
+ name = "Multilingual"
+
def __init__(self, weight: int = 1, max_retries: int = 5):
self.weight = weight
self.max_retries = max_retries
@@ -135,4 +137,4 @@ class Multilingual(BaseSingleTurnAttack):
return attack
def get_name(self) -> str:
- return "Multilingual"
+ return self.name
diff --git a/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py b/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py
index f27a6ffe..db466183 100644
--- a/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py
+++ b/deepteam/attacks/single_turn/permission_escalation/permission_escalation.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.permission_escalation.template import (
class PermissionEscalation(BaseSingleTurnAttack):
+ name = "Permission Escalation"
+
def __init__(self, weight: int = 1, max_retries: int = 3):
self.weight = weight
self.max_retries = max_retries
@@ -48,4 +50,4 @@ class PermissionEscalation(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "Permission Escalation"
+ return self.name
diff --git a/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py b/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py
index b25aa73e..6018de15 100644
--- a/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py
+++ b/deepteam/attacks/single_turn/prompt_injection/prompt_injection.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.prompt_injection.template import (
class PromptInjection(BaseSingleTurnAttack):
+ name = "Prompt Injection"
+
def __init__(self, weight: int = 1):
self.weight = weight
@@ -22,4 +24,4 @@ class PromptInjection(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "Prompt Injection"
+ return self.name
diff --git a/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py b/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py
index f51a9ed9..1ef44768 100644
--- a/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py
+++ b/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py
@@ -21,6 +21,7 @@ from deepteam.attacks.attack_simulator.utils import (
class PromptProbing(BaseSingleTurnAttack):
+ name = "Prompt Probing"
def __init__(self, weight: int = 1, max_retries: int = 3):
self.weight = weight
@@ -140,4 +141,4 @@ class PromptProbing(BaseSingleTurnAttack):
return attack
def get_name(self) -> str:
- return "Prompt Probing"
+ return self.name
diff --git a/deepteam/attacks/single_turn/roleplay/roleplay.py b/deepteam/attacks/single_turn/roleplay/roleplay.py
index e065c060..55a7e673 100644
--- a/deepteam/attacks/single_turn/roleplay/roleplay.py
+++ b/deepteam/attacks/single_turn/roleplay/roleplay.py
@@ -22,6 +22,7 @@ from deepteam.attacks.attack_simulator.utils import (
class Roleplay(BaseSingleTurnAttack):
+ name = "Roleplay"
def __init__(
self,
diff --git a/deepteam/attacks/single_turn/rot13/rot13.py b/deepteam/attacks/single_turn/rot13/rot13.py
index 85c0077d..0d8af76d 100644
--- a/deepteam/attacks/single_turn/rot13/rot13.py
+++ b/deepteam/attacks/single_turn/rot13/rot13.py
@@ -2,6 +2,8 @@ from deepteam.attacks.single_turn import BaseSingleTurnAttack
class ROT13(BaseSingleTurnAttack):
+ name = "ROT-13"
+
def __init__(self, weight: int = 1):
self.weight = weight
@@ -18,4 +20,4 @@ class ROT13(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "ROT-13"
+ return self.name
diff --git a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py
index 034c1460..c24c3f27 100644
--- a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py
+++ b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.semantic_manipulation.template import (
class LinguisticConfusion(BaseSingleTurnAttack):
+ name = "Linguistic Confusion"
+
def __init__(self, weight: int = 1, max_retries: int = 3):
self.weight = weight
self.max_retries = max_retries
@@ -43,4 +45,4 @@ class LinguisticConfusion(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "Linguistic Confusion"
+ return self.name
diff --git a/deepteam/attacks/single_turn/system_override/system_override.py b/deepteam/attacks/single_turn/system_override/system_override.py
index bd4f2709..62426f8a 100644
--- a/deepteam/attacks/single_turn/system_override/system_override.py
+++ b/deepteam/attacks/single_turn/system_override/system_override.py
@@ -7,6 +7,8 @@ from deepteam.attacks.single_turn.system_override.template import (
class SystemOverride(BaseSingleTurnAttack):
+ name = "System Override"
+
def __init__(self, weight: int = 1, max_retries: int = 3):
self.weight = weight
self.max_retries = max_retries
@@ -48,4 +50,4 @@ class SystemOverride(BaseSingleTurnAttack):
return self.enhance(attack)
def get_name(self) -> str:
- return "System Override"
+ return self.name
diff --git a/deepteam/red_teamer/api.py b/deepteam/red_teamer/api.py
index 4df12421..fa6d2fbe 100644
--- a/deepteam/red_teamer/api.py
+++ b/deepteam/red_teamer/api.py
@@ -28,9 +28,7 @@ class AttackMethodResult(BaseModel):
class APIRTTurn(TurnApi):
- turn_level_attack: Optional[str] = Field(
- alias="turnLevelAttack"
- )
+ turn_level_attack: Optional[str] = Field(alias="turnLevelAttack")
class APIRTTestCase(BaseModel):
@@ -39,9 +37,7 @@ class APIRTTestCase(BaseModel):
retrieval_context: Optional[List[str]] = Field(
None, alias="retrievalContext"
)
- tools_called: Optional[List[str]] = Field(
- None, alias="toolsCalled"
- )
+ tools_called: Optional[List[str]] = Field(None, alias="toolsCalled")
turns: Optional[List[APIRTTurn]] = Field(None)
success: Optional[bool] = Field(None)
@@ -61,14 +57,10 @@ class APIRiskAssessment(BaseModel):
vulnerability_results: List[VulnerabilityTypeResult] = Field(
alias="vulnerabilityResults"
)
- attack_results: List[AttackMethodResult] = Field(
- alias="attackResults"
- )
+ attack_results: List[AttackMethodResult] = Field(alias="attackResults")
run_duration: float = Field(alias="runDuration")
identifier: Optional[str] = Field(alias="identifier")
- assessment_cost: Optional[float] = Field(
- alias="assessmentCost"
- )
+ assessment_cost: Optional[float] = Field(alias="assessmentCost")
test_cases: List[APIRTTestCase] = Field(alias="testCases")
@@ -94,7 +86,6 @@ def map_test_case_to_api(test_case: RTTestCase, index: int) -> APIRTTestCase:
for idx, turn in enumerate(test_case.turns)
]
-
if test_case.score is not None:
success = test_case.score > 0
else:
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index b9d6deb8..7d90dad4 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -701,7 +701,6 @@ class RedTeamer:
# Pydantic version below 2.0
body = api_risk_assessment.dict(by_alias=True, exclude_none=True)
-
data, link = api.send_request(
method=HttpMethods.POST,
endpoint=Endpoints.RISK_ASSESSMENT_ENDPOINT,
diff --git a/deepteam/vulnerabilities/__init__.py b/deepteam/vulnerabilities/__init__.py
index 5fe88254..a60ba22f 100644
--- a/deepteam/vulnerabilities/__init__.py
+++ b/deepteam/vulnerabilities/__init__.py
@@ -1,3 +1,4 @@
+from typing import Dict, List
from .base_vulnerability import BaseVulnerability
from .bias.bias import Bias
from .toxicity.toxicity import Toxicity
@@ -17,13 +18,10 @@ from .competition.competition import Competition
from .graphic_content.graphic_content import GraphicContent
from .personal_safety.personal_safety import PersonalSafety
from .custom.custom import CustomVulnerability
-
-# Agentic vulnerabilities
-from .agentic.goal_theft.goal_theft import GoalTheft
-from .agentic.recursive_hijacking.recursive_hijacking import RecursiveHijacking
-from .agentic.robustness.robustness import Robustness
-from .agentic.excessive_agency.excessive_agency import ExcessiveAgency
-
+from .goal_theft.goal_theft import GoalTheft
+from .recursive_hijacking.recursive_hijacking import RecursiveHijacking
+from .robustness.robustness import Robustness
+from .excessive_agency.excessive_agency import ExcessiveAgency
__all__ = [
"BaseVulnerability",
@@ -50,33 +48,3 @@ __all__ = [
"Robustness",
"ExcessiveAgency",
]
-
-VULNERABILITY_MAP = {
- v.__name__: v
- for v in [
- Bias,
- Toxicity,
- Misinformation,
- IllegalActivity,
- PromptLeakage,
- PIILeakage,
- BFLA,
- BOLA,
- RBAC,
- DebugAccess,
- ShellInjection,
- SQLInjection,
- SSRF,
- IntellectualProperty,
- Competition,
- GraphicContent,
- PersonalSafety,
- CustomVulnerability,
- GoalTheft,
- RecursiveHijacking,
- Robustness,
- ExcessiveAgency,
- ]
-}
-
-VULNERABILITY_NAMES = sorted(list(VULNERABILITY_MAP.keys()))
diff --git a/deepteam/vulnerabilities/agentic/__init__.py b/deepteam/vulnerabilities/agentic/__init__.py
deleted file mode 100644
index 10b1cab0..00000000
--- a/deepteam/vulnerabilities/agentic/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .goal_theft.goal_theft import GoalTheft
-from .recursive_hijacking.recursive_hijacking import RecursiveHijacking
diff --git a/deepteam/vulnerabilities/base_vulnerability.py b/deepteam/vulnerabilities/base_vulnerability.py
index 5df0709d..31b98445 100644
--- a/deepteam/vulnerabilities/base_vulnerability.py
+++ b/deepteam/vulnerabilities/base_vulnerability.py
@@ -6,6 +6,8 @@ from deepteam.metrics import BaseRedTeamingMetric
class BaseVulnerability(ABC):
metric: BaseRedTeamingMetric
+ types: List[Enum]
+ name: str = None # Subclasses should override this
def __init__(self, types: List[Enum]):
"""
@@ -48,7 +50,7 @@ class BaseVulnerability(ABC):
pass
def get_name(self) -> str:
- return self.__class__.__name__
+ return self.name if self.name else self.__class__.__name__
def __repr__(self):
"""
diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.py
index 8f1c1c32..d8448262 100644
--- a/deepteam/vulnerabilities/bfla/bfla.py
+++ b/deepteam/vulnerabilities/bfla/bfla.py
@@ -25,6 +25,8 @@ BFLALiteral = Literal[
class BFLA(BaseVulnerability):
+ name = "BFLA"
+
def __init__(
self,
async_mode: bool = True,
@@ -37,7 +39,7 @@ class BFLA(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=BFLAType
+ self.name, types=types, allowed_type=BFLAType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -292,4 +294,4 @@ class BFLA(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "BFLA"
+ return self.name
diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.py
index da34f9aa..612532d8 100644
--- a/deepteam/vulnerabilities/bias/bias.py
+++ b/deepteam/vulnerabilities/bias/bias.py
@@ -20,6 +20,8 @@ BiasLiteralType = Literal["religion", "politics", "gender", "race"]
class Bias(BaseVulnerability):
+ name = "Bias"
+
def __init__(
self,
async_mode: bool = True,
@@ -34,7 +36,7 @@ class Bias(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=BiasType
+ self.name, types=types, allowed_type=BiasType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -292,4 +294,4 @@ class Bias(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Bias"
+ return self.name
diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.py
index c41d6672..7354505f 100644
--- a/deepteam/vulnerabilities/bola/bola.py
+++ b/deepteam/vulnerabilities/bola/bola.py
@@ -25,6 +25,8 @@ BOLALiteral = Literal[
class BOLA(BaseVulnerability):
+ name = "BOLA"
+
def __init__(
self,
async_mode: bool = True,
@@ -37,7 +39,7 @@ class BOLA(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=BOLAType
+ self.name, types=types, allowed_type=BOLAType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -293,4 +295,4 @@ class BOLA(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "BOLA"
+ return self.name
diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.py
index 6f911e50..cf699bd7 100644
--- a/deepteam/vulnerabilities/competition/competition.py
+++ b/deepteam/vulnerabilities/competition/competition.py
@@ -28,6 +28,8 @@ CompetitionLiteralType = Literal[
class Competition(BaseVulnerability):
+ name = "Competition"
+
def __init__(
self,
async_mode: bool = True,
@@ -42,7 +44,7 @@ class Competition(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=CompetitionType
+ self.name, types=types, allowed_type=CompetitionType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -298,4 +300,4 @@ class Competition(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Competition"
+ return self.name
diff --git a/deepteam/vulnerabilities/constants.py b/deepteam/vulnerabilities/constants.py
new file mode 100644
index 00000000..3339d3d2
--- /dev/null
+++ b/deepteam/vulnerabilities/constants.py
@@ -0,0 +1,101 @@
+from typing import Dict, List
+from .base_vulnerability import BaseVulnerability
+from .bias.bias import Bias
+from .toxicity.toxicity import Toxicity
+from .misinformation.misinformation import Misinformation
+from .illegal_activity.illegal_activity import IllegalActivity
+from .prompt_leakage.prompt_leakage import PromptLeakage
+from .pii_leakage.pii_leakage import PIILeakage
+from .bfla.bfla import BFLA
+from .bola.bola import BOLA
+from .rbac.rbac import RBAC
+from .debug_access.debug_access import DebugAccess
+from .shell_injection.shell_injection import ShellInjection
+from .sql_injection.sql_injection import SQLInjection
+from .ssrf.ssrf import SSRF
+from .intellectual_property.intellectual_property import IntellectualProperty
+from .competition.competition import Competition
+from .graphic_content.graphic_content import GraphicContent
+from .personal_safety.personal_safety import PersonalSafety
+from .goal_theft.goal_theft import GoalTheft
+from .recursive_hijacking.recursive_hijacking import RecursiveHijacking
+from .robustness.robustness import Robustness
+from .excessive_agency.excessive_agency import ExcessiveAgency
+
+
+# Import types
+from .bias.types import BiasType
+from .toxicity.types import ToxicityType
+from .misinformation.types import MisinformationType
+from .illegal_activity.types import IllegalActivityType
+from .prompt_leakage.types import PromptLeakageType
+from .pii_leakage.types import PIILeakageType
+from .bfla.types import BFLAType
+from .bola.types import BOLAType
+from .rbac.types import RBACType
+from .debug_access.types import DebugAccessType
+from .shell_injection.types import ShellInjectionType
+from .sql_injection.types import SQLInjectionType
+from .ssrf.types import SSRFType
+from .intellectual_property.types import IntellectualPropertyType
+from .competition.types import CompetitionType
+from .graphic_content.types import GraphicContentType
+from .personal_safety.types import PersonalSafetyType
+from .goal_theft.types import GoalTheftType
+from .recursive_hijacking.types import RecursiveHijackingType
+from .robustness.types import RobustnessType
+from .excessive_agency.types import ExcessiveAgencyType
+
+
+VULNERABILITY_CLASSES_MAP: Dict[str, BaseVulnerability] = {
+ v.name: v
+ for v in [
+ Bias,
+ Toxicity,
+ Misinformation,
+ IllegalActivity,
+ PromptLeakage,
+ PIILeakage,
+ BFLA,
+ BOLA,
+ RBAC,
+ DebugAccess,
+ ShellInjection,
+ SQLInjection,
+ SSRF,
+ IntellectualProperty,
+ Competition,
+ GraphicContent,
+ PersonalSafety,
+ GoalTheft,
+ RecursiveHijacking,
+ Robustness,
+ ExcessiveAgency,
+ ]
+}
+
+VULNERABILITY_TYPES_MAP: Dict[str, List[str]] = {
+ Bias.name: [e.value for e in BiasType],
+ Toxicity.name: [e.value for e in ToxicityType],
+ Misinformation.name: [e.value for e in MisinformationType],
+ IllegalActivity.name: [e.value for e in IllegalActivityType],
+ PromptLeakage.name: [e.value for e in PromptLeakageType],
+ PIILeakage.name: [e.value for e in PIILeakageType],
+ BFLA.name: [e.value for e in BFLAType],
+ BOLA.name: [e.value for e in BOLAType],
+ RBAC.name: [e.value for e in RBACType],
+ DebugAccess.name: [e.value for e in DebugAccessType],
+ ShellInjection.name: [e.value for e in ShellInjectionType],
+ SQLInjection.name: [e.value for e in SQLInjectionType],
+ SSRF.name: [e.value for e in SSRFType],
+ IntellectualProperty.name: [e.value for e in IntellectualPropertyType],
+ Competition.name: [e.value for e in CompetitionType],
+ GraphicContent.name: [e.value for e in GraphicContentType],
+ PersonalSafety.name: [e.value for e in PersonalSafetyType],
+ GoalTheft.name: [e.value for e in GoalTheftType],
+ RecursiveHijacking.name: [e.value for e in RecursiveHijackingType],
+ Robustness.name: [e.value for e in RobustnessType],
+ ExcessiveAgency.name: [e.value for e in ExcessiveAgencyType],
+}
+
+VULNERABILITY_NAMES = sorted(v.name for v in VULNERABILITY_CLASSES_MAP.values())
diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.py
index 811c5c6a..0cc54998 100644
--- a/deepteam/vulnerabilities/debug_access/debug_access.py
+++ b/deepteam/vulnerabilities/debug_access/debug_access.py
@@ -25,6 +25,8 @@ DebugAccessLiteral = Literal[
class DebugAccess(BaseVulnerability):
+ name = "Debug Access"
+
def __init__(
self,
async_mode: bool = True,
@@ -39,7 +41,7 @@ class DebugAccess(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=DebugAccessType
+ self.name, types=types, allowed_type=DebugAccessType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -292,4 +294,4 @@ class DebugAccess(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Debug Access"
+ return self.name
diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/__init__.py b/deepteam/vulnerabilities/excessive_agency/__init__.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/excessive_agency/__init__.py
rename to deepteam/vulnerabilities/excessive_agency/__init__.py
diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
similarity index 98%
rename from deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py
rename to deepteam/vulnerabilities/excessive_agency/excessive_agency.py
index edd747a4..eba5481e 100644
--- a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py
+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loop
from deepteam.utils import validate_model_callback_signature
from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.vulnerabilities.agentic.excessive_agency import (
+from deepteam.vulnerabilities.excessive_agency import (
ExcessiveAgencyType,
)
from deepteam.test_case import RTTestCase
@@ -23,6 +23,8 @@ ExcessiveAgencyLiteral = Literal["functionality", "permissions", "autonomy"]
class ExcessiveAgency(BaseVulnerability):
+ name = "Excessive Agency"
+
def __init__(
self,
async_mode: bool = True,
@@ -37,7 +39,7 @@ class ExcessiveAgency(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=ExcessiveAgencyType
+ self.name, types=types, allowed_type=ExcessiveAgencyType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -292,4 +294,4 @@ class ExcessiveAgency(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Excessive Agency"
+ return self.name
diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/template.py b/deepteam/vulnerabilities/excessive_agency/template.py
similarity index 99%
rename from deepteam/vulnerabilities/agentic/excessive_agency/template.py
rename to deepteam/vulnerabilities/excessive_agency/template.py
index de246d1e..c2a2b7f8 100644
--- a/deepteam/vulnerabilities/agentic/excessive_agency/template.py
+++ b/deepteam/vulnerabilities/excessive_agency/template.py
@@ -1,6 +1,6 @@
from typing import Optional
-from deepteam.vulnerabilities.agentic.excessive_agency.types import (
+from deepteam.vulnerabilities.excessive_agency.types import (
ExcessiveAgencyType,
)
diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/types.py b/deepteam/vulnerabilities/excessive_agency/types.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/excessive_agency/types.py
rename to deepteam/vulnerabilities/excessive_agency/types.py
diff --git a/deepteam/vulnerabilities/agentic/goal_theft/__init__.py b/deepteam/vulnerabilities/goal_theft/__init__.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/goal_theft/__init__.py
rename to deepteam/vulnerabilities/goal_theft/__init__.py
diff --git a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.py
similarity index 98%
rename from deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py
rename to deepteam/vulnerabilities/goal_theft/goal_theft.py
index 18f04889..2582b178 100644
--- a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py
+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py
@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loop
from deepteam.utils import validate_model_callback_signature
from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.vulnerabilities.agentic.goal_theft import GoalTheftType
+from deepteam.vulnerabilities.goal_theft import GoalTheftType
from deepteam.vulnerabilities.utils import validate_vulnerability_types
from deepteam.metrics.agentic.extraction_success.extraction_success import (
ExtractionSuccessMetric,
@@ -25,6 +25,8 @@ GoalTheftLiteralType = Literal[
class GoalTheft(BaseVulnerability):
+ name = "Goal Theft"
+
def __init__(
self,
async_mode: bool = True,
@@ -39,7 +41,7 @@ class GoalTheft(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=GoalTheftType
+ self.name, types=types, allowed_type=GoalTheftType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -294,4 +296,4 @@ class GoalTheft(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Goal Theft"
+ return self.name
diff --git a/deepteam/vulnerabilities/agentic/goal_theft/template.py b/deepteam/vulnerabilities/goal_theft/template.py
similarity index 99%
rename from deepteam/vulnerabilities/agentic/goal_theft/template.py
rename to deepteam/vulnerabilities/goal_theft/template.py
index b46a98ef..e3cdf4b9 100644
--- a/deepteam/vulnerabilities/agentic/goal_theft/template.py
+++ b/deepteam/vulnerabilities/goal_theft/template.py
@@ -1,6 +1,6 @@
from typing import Optional
-from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType
+from deepteam.vulnerabilities.goal_theft.types import GoalTheftType
class GoalTheftTemplate:
diff --git a/deepteam/vulnerabilities/agentic/goal_theft/types.py b/deepteam/vulnerabilities/goal_theft/types.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/goal_theft/types.py
rename to deepteam/vulnerabilities/goal_theft/types.py
diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.py
index 24ef9ae1..9ee5a3aa 100644
--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py
+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py
@@ -23,6 +23,8 @@ GraphicContentLiteral = Literal[
class GraphicContent(BaseVulnerability):
+ name = "Graphic Content"
+
def __init__(
self,
async_mode: bool = True,
@@ -37,7 +39,7 @@ class GraphicContent(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=GraphicContentType
+ self.name, types=types, allowed_type=GraphicContentType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -294,4 +296,4 @@ class GraphicContent(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Graphic Content"
+ return self.name
diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
index 003ec6b3..e28cc246 100644
--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
@@ -29,6 +29,8 @@ IllegalActivityLiteral = Literal[
class IllegalActivity(BaseVulnerability):
+ name = "Illegal Activity"
+
def __init__(
self,
async_mode: bool = True,
@@ -43,7 +45,7 @@ class IllegalActivity(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=IllegalActivityType
+ self.name, types=types, allowed_type=IllegalActivityType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -301,4 +303,4 @@ class IllegalActivity(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Illegal Activity"
+ return self.name
diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
index 258b9a5c..6f34c29f 100644
--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
@@ -28,6 +28,8 @@ IntellectualPropertyLiteral = Literal[
class IntellectualProperty(BaseVulnerability):
+ name = "Intellectual Property"
+
def __init__(
self,
async_mode: bool = True,
@@ -42,7 +44,7 @@ class IntellectualProperty(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=IntellectualPropertyType
+ self.name, types=types, allowed_type=IntellectualPropertyType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -299,4 +301,4 @@ class IntellectualProperty(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Intellectual Property"
+ return self.name
diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.py
index df57acbc..12698444 100644
--- a/deepteam/vulnerabilities/misinformation/misinformation.py
+++ b/deepteam/vulnerabilities/misinformation/misinformation.py
@@ -24,6 +24,8 @@ MisinformationLiteral = Literal[
class Misinformation(BaseVulnerability):
+ name = "Misinformation"
+
def __init__(
self,
async_mode: bool = True,
@@ -38,7 +40,7 @@ class Misinformation(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=MisinformationType
+ self.name, types=types, allowed_type=MisinformationType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -294,4 +296,4 @@ class Misinformation(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Misinformation"
+ return self.name
diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.py
index 3bbb4e89..8be4d671 100644
--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py
+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py
@@ -27,6 +27,8 @@ PersonalSafetyLiteral = Literal[
class PersonalSafety(BaseVulnerability):
+ name = "Personal Safety"
+
def __init__(
self,
async_mode: bool = True,
@@ -41,7 +43,7 @@ class PersonalSafety(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=PersonalSafetyType
+ self.name, types=types, allowed_type=PersonalSafetyType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -293,4 +295,4 @@ class PersonalSafety(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Personal Safety"
+ return self.name
diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
index e0980623..2d7d61e6 100644
--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
@@ -26,6 +26,8 @@ PIILeakageLiteral = Literal[
class PIILeakage(BaseVulnerability):
+ name = "PII Leakage"
+
def __init__(
self,
async_mode: bool = True,
@@ -40,7 +42,7 @@ class PIILeakage(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=PIILeakageType
+ self.name, types=types, allowed_type=PIILeakageType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -300,4 +302,4 @@ class PIILeakage(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "PII Leakage"
+ return self.name
diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
index 2d81deef..8bf44b2e 100644
--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
@@ -26,6 +26,8 @@ PromptLeakageLiteral = Literal[
class PromptLeakage(BaseVulnerability):
+ name = "Prompt Leakage"
+
def __init__(
self,
async_mode: bool = True,
@@ -40,7 +42,7 @@ class PromptLeakage(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=PromptLeakageType
+ self.name, types=types, allowed_type=PromptLeakageType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -297,4 +299,4 @@ class PromptLeakage(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Prompt Leakage"
+ return self.name
diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.py
index f13c73e1..233983de 100644
--- a/deepteam/vulnerabilities/rbac/rbac.py
+++ b/deepteam/vulnerabilities/rbac/rbac.py
@@ -25,6 +25,8 @@ RBACLiteral = Literal[
class RBAC(BaseVulnerability):
+ name = "RBAC"
+
def __init__(
self,
async_mode: bool = True,
@@ -37,7 +39,7 @@ class RBAC(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=RBACType
+ self.name, types=types, allowed_type=RBACType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -294,4 +296,4 @@ class RBAC(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "RBAC"
+ return self.name
diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/__init__.py b/deepteam/vulnerabilities/recursive_hijacking/__init__.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/recursive_hijacking/__init__.py
rename to deepteam/vulnerabilities/recursive_hijacking/__init__.py
diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
similarity index 98%
rename from deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py
rename to deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
index b04c73ea..000db93a 100644
--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py
+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loop
from deepteam.utils import validate_model_callback_signature
from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.vulnerabilities.agentic.recursive_hijacking import (
+from deepteam.vulnerabilities.recursive_hijacking import (
RecursiveHijackingType,
)
from deepteam.vulnerabilities.utils import validate_vulnerability_types
@@ -29,6 +29,8 @@ RecursiveHijackingLiteralType = Literal[
class RecursiveHijacking(BaseVulnerability):
+ name = "Recursive Hijacking"
+
def __init__(
self,
async_mode: bool = True,
@@ -43,7 +45,7 @@ class RecursiveHijacking(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=RecursiveHijackingType
+ self.name, types=types, allowed_type=RecursiveHijackingType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -298,4 +300,4 @@ class RecursiveHijacking(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Recursive Hijacking"
+ return self.name
diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/template.py b/deepteam/vulnerabilities/recursive_hijacking/template.py
similarity index 99%
rename from deepteam/vulnerabilities/agentic/recursive_hijacking/template.py
rename to deepteam/vulnerabilities/recursive_hijacking/template.py
index 76bc3a1a..2b7b93cf 100644
--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/template.py
+++ b/deepteam/vulnerabilities/recursive_hijacking/template.py
@@ -1,6 +1,6 @@
from typing import Optional
-from deepteam.vulnerabilities.agentic.recursive_hijacking.types import (
+from deepteam.vulnerabilities.recursive_hijacking.types import (
RecursiveHijackingType,
)
diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/types.py b/deepteam/vulnerabilities/recursive_hijacking/types.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/recursive_hijacking/types.py
rename to deepteam/vulnerabilities/recursive_hijacking/types.py
diff --git a/deepteam/vulnerabilities/agentic/robustness/__init__.py b/deepteam/vulnerabilities/robustness/__init__.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/robustness/__init__.py
rename to deepteam/vulnerabilities/robustness/__init__.py
diff --git a/deepteam/vulnerabilities/agentic/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.py
similarity index 98%
rename from deepteam/vulnerabilities/agentic/robustness/robustness.py
rename to deepteam/vulnerabilities/robustness/robustness.py
index 3879e494..5095f923 100644
--- a/deepteam/vulnerabilities/agentic/robustness/robustness.py
+++ b/deepteam/vulnerabilities/robustness/robustness.py
@@ -8,7 +8,7 @@ from deepeval.utils import get_or_create_event_loop
from deepteam.utils import validate_model_callback_signature
from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.vulnerabilities.agentic.robustness import RobustnessType
+from deepteam.vulnerabilities.robustness import RobustnessType
from deepteam.vulnerabilities.utils import validate_vulnerability_types
from deepteam.metrics import (
HijackingMetric,
@@ -24,6 +24,8 @@ RobustnessLiteral = Literal["input_overreliance", "hijacking"]
class Robustness(BaseVulnerability):
+ name = "Robustness"
+
def __init__(
self,
async_mode: bool = True,
@@ -38,7 +40,7 @@ class Robustness(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=RobustnessType
+ self.name, types=types, allowed_type=RobustnessType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -305,4 +307,4 @@ class Robustness(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Robustness"
+ return self.name
diff --git a/deepteam/vulnerabilities/agentic/robustness/template.py b/deepteam/vulnerabilities/robustness/template.py
similarity index 99%
rename from deepteam/vulnerabilities/agentic/robustness/template.py
rename to deepteam/vulnerabilities/robustness/template.py
index b6d72ff6..a0bdf1a7 100644
--- a/deepteam/vulnerabilities/agentic/robustness/template.py
+++ b/deepteam/vulnerabilities/robustness/template.py
@@ -1,6 +1,6 @@
from typing import Optional
-from deepteam.vulnerabilities.agentic.robustness.types import RobustnessType
+from deepteam.vulnerabilities.robustness.types import RobustnessType
class RobustnessTemplate:
diff --git a/deepteam/vulnerabilities/agentic/robustness/types.py b/deepteam/vulnerabilities/robustness/types.py
similarity index 100%
rename from deepteam/vulnerabilities/agentic/robustness/types.py
rename to deepteam/vulnerabilities/robustness/types.py
diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.py
index 35413b16..3e1386b9 100644
--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py
+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py
@@ -25,6 +25,8 @@ ShellInjectionLiteral = Literal[
class ShellInjection(BaseVulnerability):
+ name = "Shell Injection"
+
def __init__(
self,
async_mode: bool = True,
@@ -39,7 +41,7 @@ class ShellInjection(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=ShellInjectionType
+ self.name, types=types, allowed_type=ShellInjectionType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -289,4 +291,4 @@ class ShellInjection(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Shell Injection"
+ return self.name
diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.py
index 705c3de5..e888f55e 100644
--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py
+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py
@@ -25,6 +25,8 @@ SQLInjectionLiteral = Literal[
class SQLInjection(BaseVulnerability):
+ name = "SQL Injection"
+
def __init__(
self,
async_mode: bool = True,
@@ -39,7 +41,7 @@ class SQLInjection(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=SQLInjectionType
+ self.name, types=types, allowed_type=SQLInjectionType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -289,4 +291,4 @@ class SQLInjection(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "SQL Injection"
+ return self.name
diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.py
index df2e279b..e80730ac 100644
--- a/deepteam/vulnerabilities/ssrf/ssrf.py
+++ b/deepteam/vulnerabilities/ssrf/ssrf.py
@@ -25,6 +25,8 @@ SSRFLiteral = Literal[
class SSRF(BaseVulnerability):
+ name = "SSRF"
+
def __init__(
self,
async_mode: bool = True,
@@ -37,7 +39,7 @@ class SSRF(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=SSRFType
+ self.name, types=types, allowed_type=SSRFType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -288,4 +290,4 @@ class SSRF(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "SSRF"
+ return self.name
diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.py
index 36bfde3d..ef5464bf 100644
--- a/deepteam/vulnerabilities/toxicity/toxicity.py
+++ b/deepteam/vulnerabilities/toxicity/toxicity.py
@@ -21,6 +21,8 @@ ToxicityLiteral = Literal["profanity", "insults", "threats", "mockery"]
class Toxicity(BaseVulnerability):
+ name = "Toxicity"
+
def __init__(
self,
async_mode: bool = True,
@@ -35,7 +37,7 @@ class Toxicity(BaseVulnerability):
purpose: Optional[str] = None,
):
enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=ToxicityType
+ self.name, types=types, allowed_type=ToxicityType
)
self.async_mode = async_mode
self.verbose_mode = verbose_mode
@@ -289,4 +291,4 @@ class Toxicity(BaseVulnerability):
return self.vulnerable
def get_name(self) -> str:
- return "Toxicity"
+ return self.name
diff --git a/deepteam/vulnerabilities/types.py b/deepteam/vulnerabilities/types.py
index bd54ccfd..94b04de8 100644
--- a/deepteam/vulnerabilities/types.py
+++ b/deepteam/vulnerabilities/types.py
@@ -50,26 +50,26 @@ from deepteam.vulnerabilities.shell_injection.template import (
ShellInjectionTemplate,
)
from deepteam.vulnerabilities.sql_injection.template import SQLInjectionTemplate
-from deepteam.vulnerabilities.agentic.robustness import RobustnessType
-from deepteam.vulnerabilities.agentic.robustness.template import (
+from deepteam.vulnerabilities.robustness import RobustnessType
+from deepteam.vulnerabilities.robustness.template import (
RobustnessTemplate,
)
-from deepteam.vulnerabilities.agentic.excessive_agency import (
+from deepteam.vulnerabilities.excessive_agency import (
ExcessiveAgencyType,
)
-from deepteam.vulnerabilities.agentic.excessive_agency.template import (
+from deepteam.vulnerabilities.excessive_agency.template import (
ExcessiveAgencyTemplate,
)
# Import agentic vulnerability types
-from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType
-from deepteam.vulnerabilities.agentic.recursive_hijacking.types import (
+from deepteam.vulnerabilities.goal_theft.types import GoalTheftType
+from deepteam.vulnerabilities.recursive_hijacking.types import (
RecursiveHijackingType,
)
-from deepteam.vulnerabilities.agentic.goal_theft.template import (
+from deepteam.vulnerabilities.goal_theft.template import (
GoalTheftTemplate,
)
-from deepteam.vulnerabilities.agentic.recursive_hijacking.template import (
+from deepteam.vulnerabilities.recursive_hijacking.template import (
RecursiveHijackingTemplate,
)
diff --git a/kk.py b/kk.py
index a3c41c32..c62a4d97 100644
--- a/kk.py
+++ b/kk.py
@@ -1,17 +1,39 @@
from deepteam import red_team
-from deepteam.vulnerabilities import Bias
+from deepteam.vulnerabilities import Bias, BOLA
from deepteam.attacks.single_turn import Roleplay
import os
-async def your_callback(input: str) -> str:
- return "I'm sorry but I can't answer this: " + input
+# async def your_callback(input: str) -> str:
+# return "I'm sorry but I can't answer this: " + input
-print(os.environ.get("CONFIDENT_API_KEY"))
+# print(os.environ.get("CONFIDENT_API_KEY"))
-risk_assessment = red_team(
- attacks=[Roleplay()],
- vulnerabilities=[Bias()],
- model_callback=your_callback,
+# risk_assessment = red_team(
+# attacks=[Roleplay()],
+# vulnerabilities=[Bias(), BOLA()],
+# model_callback=your_callback,
+# )
+
+from deepteam.vulnerabilities.constants import (
+ VULNERABILITY_NAMES,
+ VULNERABILITY_CLASSES_MAP,
+ VULNERABILITY_TYPES_MAP,
+)
+from deepteam.attacks.single_turn.constants import (
+ SINGLE_TURN_ATTACK_NAMES,
+ SINGLE_TURN_ATTACK_CLASSES_MAP,
)
+from deepteam.attacks.multi_turn.constants import (
+ MULTI_TURN_ATTACK_NAMES,
+ MULTI_TURN_ATTACK_CLASSES_MAP,
+)
+
+print(SINGLE_TURN_ATTACK_NAMES, "@@@")
+print(SINGLE_TURN_ATTACK_CLASSES_MAP, "@@@")
+print(MULTI_TURN_ATTACK_NAMES, "@@@")
+print(MULTI_TURN_ATTACK_CLASSES_MAP, "@@@")
+print(VULNERABILITY_NAMES, "@@@")
+print(VULNERABILITY_CLASSES_MAP, "@@@")
+print(VULNERABILITY_TYPES_MAP, "@@@")