DeepTeam is a framework to red team LLMs and AI agents.
fix attacks
penguine7750ed7b8262
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex 8bc22e71..ef755531 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -264,25 +264,11 @@ class AttackSimulator:test_case: RTTestCase,ignore_errors: bool,):- from deepteam.attacks.multi_turn import (- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- SequentialJailbreak,- TreeJailbreaking,- )from deepteam.test_case.test_case import RTTurn-- MULTI_TURN_ATTACKS = [- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- TreeJailbreaking,- SequentialJailbreak,- ]+ from deepteam.attacks.multi_turn import BaseMultiTurnAttack# Check if the attack is a multi-turn attack- if type(attack) in MULTI_TURN_ATTACKS:+ if isinstance(attack, BaseMultiTurnAttack):# This is a multi-turn attackattack_input = test_case.inputif attack_input is None:@@ -375,23 +361,11 @@ class AttackSimulator:ignore_errors: bool,):from deepteam.attacks.multi_turn import (- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- SequentialJailbreak,- TreeJailbreaking,+ BaseMultiTurnAttack,)from deepteam.test_case.test_case import RTTurn- MULTI_TURN_ATTACKS = [- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- TreeJailbreaking,- SequentialJailbreak,- ]-- if type(attack) in MULTI_TURN_ATTACKS:+ if isinstance(attack, BaseMultiTurnAttack):# This is multi-turn attackattack_input = test_case.inputif attack_input is None:diff --git a/deepteam/attacks/multi_turn/__init__.py b/deepteam/attacks/multi_turn/__init__.pyindex 95163e5f..4583286a 100644--- a/deepteam/attacks/multi_turn/__init__.py+++ b/deepteam/attacks/multi_turn/__init__.py@@ -3,6 +3,7 @@ from .linear_jailbreaking import LinearJailbreakingfrom .tree_jailbreaking import TreeJailbreakingfrom .sequential_break import SequentialJailbreakfrom .bad_likert_judge import BadLikertJudge+from .base_multi_turn_attack import BaseMultiTurnAttack__all__ = ["CrescendoJailbreaking",@@ -10,4 +11,5 @@ __all__ = ["TreeJailbreaking","SequentialJailbreak","BadLikertJudge",+ "BaseMultiTurnAttack",]diff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.pyindex 38899726..c26a90e0 100644--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py@@ -1,9 +1,9 @@-from pydantic import BaseModelfrom typing import Optional, Union, List, Dictfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_modelimport random-from deepteam.attacks import BaseAttack++from deepteam.attacks.multi_turn import BaseMultiTurnAttackfrom deepteam.attacks.multi_turn.bad_likert_judge.schema import (BLJJudge,)@@ -24,7 +24,7 @@ from deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability-class BadLikertJudge(BaseAttack):+class BadLikertJudge(BaseMultiTurnAttack):def __init__(self,weight: int = 1,diff --git a/deepteam/attacks/multi_turn/base_multi_turn_attack.py b/deepteam/attacks/multi_turn/base_multi_turn_attack.pynew file mode 100644index 00000000..e74d17d2--- /dev/null+++ b/deepteam/attacks/multi_turn/base_multi_turn_attack.py@@ -0,0 +1,5 @@+from deepteam.attacks.base_attack import BaseAttack+++class BaseMultiTurnAttack(BaseAttack):+ passdiff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.pyindex 759526f9..d67fcbe4 100644--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py@@ -1,5 +1,4 @@from typing import List, Dict, Any, Optional, Tuple, Union-from pydantic import BaseModelfrom uuid import uuid4import jsonimport random@@ -7,9 +6,10 @@ import randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model+from deepteam.attacks.single_turn import BaseSingleTurnAttackfrom deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attack-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn import BaseMultiTurnAttackfrom deepteam.attacks.multi_turn.crescendo_jailbreaking.template import (JailBreakingCrescendoTemplate,)@@ -50,14 +50,14 @@ class MemorySystem:return new_conversation_id-class CrescendoJailbreaking(BaseAttack):+class CrescendoJailbreaking(BaseMultiTurnAttack):def __init__(self,weight: int = 1,max_rounds: int = 10,max_backtracks: int = 10,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weightdiff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.pyindex e1a73747..c64ad163 100644--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py@@ -18,7 +18,8 @@ from deepteam.attacks.attack_simulator.utils import ()from deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.utils import create_progress, update_pbar, add_pbar-from deepteam.attacks import BaseAttack+from deepteam.attacks.single_turn import BaseSingleTurnAttack+from deepteam.attacks.multi_turn import BaseMultiTurnAttackfrom deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attackfrom deepteam.attacks.multi_turn.base_schema import NonRefusalfrom deepteam.attacks.multi_turn.base_template import BaseMultiTurnTemplate@@ -28,12 +29,12 @@ from deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability-class LinearJailbreaking(BaseAttack):+class LinearJailbreaking(BaseMultiTurnAttack):def __init__(self,weight: int = 1,num_turns: int = 5,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weightdiff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.pyindex 5715308e..57673bc9 100644--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py@@ -4,7 +4,9 @@ import randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model-from deepteam.attacks import BaseAttack+from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+)from deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks.multi_turn.sequential_break.schema import (RewrittenDialogue,@@ -27,16 +29,17 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusalfrom deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability+from deepteam.attacks.single_turn import BaseSingleTurnAttack-class SequentialJailbreak(BaseAttack):+class SequentialJailbreak(BaseMultiTurnAttack):def __init__(self,weight: int = 1,type: Optional[SequentialJailbreakTypeLiteral] = None,persona: Optional[DialogueTypeLiteral] = None,num_turns: int = 5,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weightdiff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.pyindex bbede830..271a2e72 100644--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py@@ -27,6 +27,8 @@ from deepteam.attacks.multi_turn.base_schema import NonRefusalfrom deepteam.test_case.test_case import RTTurnfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.vulnerabilities import BaseVulnerability+from deepteam.attacks.multi_turn import BaseMultiTurnAttack+from deepteam.attacks.single_turn import BaseSingleTurnAttackclass TreeNode:@@ -48,13 +50,13 @@ class TreeNode:self.conversation_history = conversation_history or []-class TreeJailbreaking(BaseAttack):+class TreeJailbreaking(BaseMultiTurnAttack):def __init__(self,weight: int = 1,max_depth: int = 5,- turn_level_attacks: Optional[List[BaseAttack]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = "gpt-4o-mini",):self.weight = weight