Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

fix attacks

penguine7750ed7b8262
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.py
index 8bc22e71..ef755531 100644
--- a/deepteam/attacks/attack_simulator/attack_simulator.py
+++ b/deepteam/attacks/attack_simulator/attack_simulator.py
@@ -264,25 +264,11 @@ class AttackSimulator:
test_case: RTTestCase,
ignore_errors: bool,
):
- from deepteam.attacks.multi_turn import (
- BadLikertJudge,
- CrescendoJailbreaking,
- LinearJailbreaking,
- SequentialJailbreak,
- TreeJailbreaking,
- )
from deepteam.test_case.test_case import RTTurn
-
- MULTI_TURN_ATTACKS = [
- BadLikertJudge,
- CrescendoJailbreaking,
- LinearJailbreaking,
- TreeJailbreaking,
- SequentialJailbreak,
- ]
+ from deepteam.attacks.multi_turn import BaseMultiTurnAttack
# Check if the attack is a multi-turn attack
- if type(attack) in MULTI_TURN_ATTACKS:
+ if isinstance(attack, BaseMultiTurnAttack):
# This is a multi-turn attack
attack_input = test_case.input
if attack_input is None:
@@ -375,23 +361,11 @@ class AttackSimulator:
ignore_errors: bool,
):
from deepteam.attacks.multi_turn import (
- BadLikertJudge,
- CrescendoJailbreaking,
- LinearJailbreaking,
- SequentialJailbreak,
- TreeJailbreaking,
+ BaseMultiTurnAttack,
)
from deepteam.test_case.test_case import RTTurn
- MULTI_TURN_ATTACKS = [
- BadLikertJudge,
- CrescendoJailbreaking,
- LinearJailbreaking,
- TreeJailbreaking,
- SequentialJailbreak,
- ]
-
- if type(attack) in MULTI_TURN_ATTACKS:
+ if isinstance(attack, BaseMultiTurnAttack):
# This is multi-turn attack
attack_input = test_case.input
if attack_input is None:
diff --git a/deepteam/attacks/multi_turn/__init__.py b/deepteam/attacks/multi_turn/__init__.py
index 95163e5f..4583286a 100644
--- a/deepteam/attacks/multi_turn/__init__.py
+++ b/deepteam/attacks/multi_turn/__init__.py
@@ -3,6 +3,7 @@ from .linear_jailbreaking import LinearJailbreaking
from .tree_jailbreaking import TreeJailbreaking
from .sequential_break import SequentialJailbreak
from .bad_likert_judge import BadLikertJudge
+from .base_multi_turn_attack import BaseMultiTurnAttack
__all__ = [
"CrescendoJailbreaking",
@@ -10,4 +11,5 @@ __all__ = [
"TreeJailbreaking",
"SequentialJailbreak",
"BadLikertJudge",
+ "BaseMultiTurnAttack",
]
diff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
index 38899726..c26a90e0 100644
--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py
@@ -1,9 +1,9 @@
-from pydantic import BaseModel
from typing import Optional, Union, List, Dict
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
import random
-from deepteam.attacks import BaseAttack
+
+from deepteam.attacks.multi_turn import BaseMultiTurnAttack
from deepteam.attacks.multi_turn.bad_likert_judge.schema import (
BLJJudge,
)
@@ -24,7 +24,7 @@ from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
-class BadLikertJudge(BaseAttack):
+class BadLikertJudge(BaseMultiTurnAttack):
def __init__(
self,
weight: int = 1,
diff --git a/deepteam/attacks/multi_turn/base_multi_turn_attack.py b/deepteam/attacks/multi_turn/base_multi_turn_attack.py
new file mode 100644
index 00000000..e74d17d2
--- /dev/null
+++ b/deepteam/attacks/multi_turn/base_multi_turn_attack.py
@@ -0,0 +1,5 @@
+from deepteam.attacks.base_attack import BaseAttack
+
+
+class BaseMultiTurnAttack(BaseAttack):
+ pass
diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
index 759526f9..d67fcbe4 100644
--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py
@@ -1,5 +1,4 @@
from typing import List, Dict, Any, Optional, Tuple, Union
-from pydantic import BaseModel
from uuid import uuid4
import json
import random
@@ -7,9 +6,10 @@ import random
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
+from deepteam.attacks.single_turn import BaseSingleTurnAttack
from deepteam.utils import create_progress, update_pbar, add_pbar
from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn import BaseMultiTurnAttack
from deepteam.attacks.multi_turn.crescendo_jailbreaking.template import (
JailBreakingCrescendoTemplate,
)
@@ -50,14 +50,14 @@ class MemorySystem:
return new_conversation_id
-class CrescendoJailbreaking(BaseAttack):
+class CrescendoJailbreaking(BaseMultiTurnAttack):
def __init__(
self,
weight: int = 1,
max_rounds: int = 10,
max_backtracks: int = 10,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
index e1a73747..c64ad163 100644
--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py
@@ -18,7 +18,8 @@ from deepteam.attacks.attack_simulator.utils import (
)
from deepteam.attacks.multi_turn.types import CallbackType
from deepteam.utils import create_progress, update_pbar, add_pbar
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.single_turn import BaseSingleTurnAttack
+from deepteam.attacks.multi_turn import BaseMultiTurnAttack
from deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack
from deepteam.attacks.multi_turn.base_schema import NonRefusal
from deepteam.attacks.multi_turn.base_template import BaseMultiTurnTemplate
@@ -28,12 +29,12 @@ from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
-class LinearJailbreaking(BaseAttack):
+class LinearJailbreaking(BaseMultiTurnAttack):
def __init__(
self,
weight: int = 1,
num_turns: int = 5,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
diff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
index 5715308e..57673bc9 100644
--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py
@@ -4,7 +4,9 @@ import random
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
-from deepteam.attacks import BaseAttack
+from deepteam.attacks.multi_turn.base_multi_turn_attack import (
+ BaseMultiTurnAttack,
+)
from deepteam.utils import create_progress, update_pbar, add_pbar
from deepteam.attacks.multi_turn.sequential_break.schema import (
RewrittenDialogue,
@@ -27,16 +29,17 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusal
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
+from deepteam.attacks.single_turn import BaseSingleTurnAttack
-class SequentialJailbreak(BaseAttack):
+class SequentialJailbreak(BaseMultiTurnAttack):
def __init__(
self,
weight: int = 1,
type: Optional[SequentialJailbreakTypeLiteral] = None,
persona: Optional[DialogueTypeLiteral] = None,
num_turns: int = 5,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight
diff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
index bbede830..271a2e72 100644
--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py
@@ -27,6 +27,8 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusal
from deepteam.test_case.test_case import RTTurn
from deepteam.vulnerabilities.types import VulnerabilityType
from deepteam.vulnerabilities import BaseVulnerability
+from deepteam.attacks.multi_turn import BaseMultiTurnAttack
+from deepteam.attacks.single_turn import BaseSingleTurnAttack
class TreeNode:
@@ -48,13 +50,13 @@ class TreeNode:
self.conversation_history = conversation_history or []
-class TreeJailbreaking(BaseAttack):
+class TreeJailbreaking(BaseMultiTurnAttack):
def __init__(
self,
weight: int = 1,
max_depth: int = 5,
- turn_level_attacks: Optional[List[BaseAttack]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",
):
self.weight = weight