DeepTeam is a framework to red team LLMs and AI agents.
fix simulator model
penguine8d98d167527e
diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.pyindex b2c84e89..5b3c15e6 100644--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py@@ -51,15 +51,11 @@ class CrescendoJailbreaking(BaseAttack):def __init__(self,model_callback: CallbackType,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,weight: int = 1,max_rounds: int = 10,max_backtracks: int = 10,):self.model_callback = model_callback- self.simulator_model = simulator_model- self.using_native_model = using_native_modelself.weight = weightself.memory = MemorySystem()@@ -68,11 +64,9 @@ class CrescendoJailbreaking(BaseAttack):self.max_rounds = max_roundsself.max_backtracks = max_backtracks- ##################################################- ### Enhance ######################################- ##################################################+ def enhance(self, attack: str, simulator_model: DeepEvalBaseLLM) -> str:+ self.simulator_model = simulator_model- def enhance(self, attack: str) -> str:round_num = 0backtrack_count = 0last_response = ""@@ -150,8 +144,10 @@ class CrescendoJailbreaking(BaseAttack):return enhanced_attackasync def a_enhance(- self, attack: str, max_rounds: int = 10, max_backtracks: int = 10+ self, attack: str, simulator_model: DeepEvalBaseLLM) -> str:+ self.simulator_model = simulator_model+round_num = 0backtrack_count = 0last_response = ""@@ -161,7 +157,7 @@ class CrescendoJailbreaking(BaseAttack):system_prompt = JailBreakingCrescendoTemplate.crescendo_system_prompt(conversation_objective=attack,current_round=round_num + 1,- max_rounds=max_rounds,+ max_rounds=self.max_rounds,)self.memory.add_message(self.red_teaming_chat_conversation_id,@@ -169,19 +165,19 @@ class CrescendoJailbreaking(BaseAttack):)pbar_rounds = tqdm(- total=max_rounds * 4,+ total=self.max_rounds * 4,desc="...... πΊ Crescendo Jailbreaking: Rounds",leave=False,)# Inner progress bar: Tracks the number of backtrackspbar_backtracks = tqdm(- total=max_backtracks,+ total=self.max_backtracks,desc="...... πΊ Crescendo Jailbreaking: BackTracking",leave=False,)- while round_num < max_rounds:+ while round_num < self.max_rounds:round_num += 1enhanced_attack = await self.a_generate_attack(round_num, eval_flag, last_response@@ -212,11 +208,11 @@ class CrescendoJailbreaking(BaseAttack):)eval_flag = Falseeval_percentage = None- if backtrack_count < max_backtracks:+ if backtrack_count < self.max_backtracks:round_num -= 1pbar_rounds.update(-3)pbar_backtracks.update(1)- if backtrack_count >= max_backtracks:+ if backtrack_count >= self.max_backtracks:breakcontinue@@ -405,13 +401,11 @@ class CrescendoJailbreaking(BaseAttack):##################################################def _generate_schema(self, prompt: str, schema: BaseModel):- return generate_schema(- prompt, schema, self.using_native_model, self.simulator_model- )+ return generate_schema(prompt, schema, False, self.simulator_model)async def _a_generate_schema(self, prompt: str, schema: BaseModel):return await a_generate_schema(- prompt, schema, self.using_native_model, self.simulator_model+ prompt, schema, False, self.simulator_model)def get_name(self) -> str:diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.pyindex 4be35a3a..cec0cda7 100644--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py@@ -25,20 +25,19 @@ from deepteam.attacks.multi_turn.types import CallbackTypeclass LinearJailbreaking(BaseAttack):def __init__(self,- model_callback: CallbackType,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,weight: int = 1,turns: int = 5,):- self.model_callback = model_callback- self.simulator_model = simulator_model- self.using_native_model = using_native_modelself.weight = weightself.turns = turns- def enhance(self, attack: str) -> str:- """Enhance the attack using linear jailbreaking with multiple iterations synchronously."""+ def enhance(+ self,+ attack: str,+ model_callback: CallbackType,+ simulator_model: DeepEvalBaseLLM,+ ) -> str:+ self.simulator_model = simulator_model# Define Progress Barllm_calls_per_iteration = 5@@ -56,20 +55,20 @@ class LinearJailbreaking(BaseAttack):for _ in range(self.turns):# 1st step: Enhance the initial attackconversation = json.dumps(conversation_json)- res: ImprovementPrompt = self._generate_schema(+ improvement_res: ImprovementPrompt = self._generate_schema(conversation, ImprovementPrompt)- enhanced_attack = res.prompt+ enhanced_attack = improvement_res.promptpbar.update(1)# 2nd step: Check if enhanced attack is a refusal statementnon_refusal_prompt = JailBreakingTemplate.non_refusal(attack, enhanced_attack)- res: NonRefusal = self._generate_schema(+ non_refusal_res: NonRefusal = self._generate_schema(non_refusal_prompt, NonRefusal)- classification = res.classification+ classification = non_refusal_res.classificationif classification == "Refusal":pbar.n = pbar.totalpbar.close()@@ -82,13 +81,15 @@ class LinearJailbreaking(BaseAttack):is_on_topic_prompt = JailBreakingTemplate.on_topic(attack, current_attack)- res: OnTopic = self._generate_schema(is_on_topic_prompt, OnTopic)+ on_topic_res: OnTopic = self._generate_schema(+ is_on_topic_prompt, OnTopic+ )- on_topic = res.on_topic+ on_topic = on_topic_res.on_topicpbar.update(1)# 4th step: Generate a response from the target LLM using the enhanced attack- target_response = self.model_callback(current_attack)+ target_response = model_callback(current_attack)penalized_phrases = ["in the face of impending doom"]contains_penalized_phrase = any(@@ -101,9 +102,9 @@ class LinearJailbreaking(BaseAttack):judge_prompt = JailBreakingTemplate.linear_judge(attack, current_attack, target_response)- res: Rating = self._generate_schema(judge_prompt, Rating)+ rating_res: Rating = self._generate_schema(judge_prompt, Rating)- score = res.rating+ score = rating_res.ratingpbar.update(1)# Apply penalty if the phrase is present@@ -125,8 +126,13 @@ class LinearJailbreaking(BaseAttack):pbar.close()return current_attack- async def a_enhance(self, attack: str) -> str:- """Enhance the attack using linear jailbreaking with multiple iterations asynchronously."""+ async def a_enhance(+ self,+ attack: str,+ model_callback: CallbackType,+ simulator_model: DeepEvalBaseLLM,+ ) -> str:+ self.simulator_model = simulator_model# Define Progress Barllm_calls_per_iteration = 5@@ -144,20 +150,20 @@ class LinearJailbreaking(BaseAttack):for _ in range(self.turns):# 1st step: Enhance the initial attackconversation = json.dumps(conversation_json)- res: ImprovementPrompt = await self._a_generate_schema(+ improvement_res: ImprovementPrompt = await self._a_generate_schema(conversation, ImprovementPrompt)- enhanced_attack = res.prompt+ enhanced_attack = improvement_res.promptpbar.update(1)# 2nd step: Check if enhanced attack is a refusal statementnon_refusal_prompt = JailBreakingTemplate.non_refusal(attack, enhanced_attack)- res: NonRefusal = await self._a_generate_schema(+ non_refusal_res: NonRefusal = await self._a_generate_schema(non_refusal_prompt, NonRefusal)- classification = res.classification+ classification = non_refusal_res.classificationif classification == "Refusal":pbar.n = pbar.totalpbar.close()@@ -170,14 +176,14 @@ class LinearJailbreaking(BaseAttack):is_on_topic_prompt = JailBreakingTemplate.on_topic(attack, current_attack)- res: OnTopic = await self._a_generate_schema(+ on_topic_res: OnTopic = await self._a_generate_schema(is_on_topic_prompt, OnTopic)- on_topic = res.on_topic+ on_topic = on_topic_res.on_topicpbar.update(1)# 4th step: Generate a response from the target LLM using the enhanced attack- target_response = await self.model_callback(current_attack)+ target_response = await model_callback(current_attack)penalized_phrases = ["in the face of impending doom"]contains_penalized_phrase = any(@@ -190,8 +196,10 @@ class LinearJailbreaking(BaseAttack):judge_prompt = JailBreakingTemplate.linear_judge(attack, current_attack, target_response)- res: Rating = await self._a_generate_schema(judge_prompt, Rating)- score = res.rating+ rating_res: Rating = await self._a_generate_schema(+ judge_prompt, Rating+ )+ score = rating_res.ratingpbar.update(1)# Apply penalty if the phrase is present@@ -218,13 +226,11 @@ class LinearJailbreaking(BaseAttack):##################################################def _generate_schema(self, prompt: str, schema: BaseModel):- return generate_schema(- prompt, schema, self.using_native_model, self.simulator_model- )+ return generate_schema(prompt, schema, False, self.simulator_model)async def _a_generate_schema(self, prompt: str, schema: BaseModel):return await a_generate_schema(- prompt, schema, self.using_native_model, self.simulator_model+ prompt, schema, False, self.simulator_model)def get_name(self) -> str:diff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.pyindex 98687689..db1abdd5 100644--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py@@ -39,22 +39,19 @@ class TreeJailbreaking(BaseAttack):def __init__(self,- model_callback: CallbackType,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,weight: int = 1,):- self.model_callback = model_callback- self.simulator_model = simulator_model- self.using_native_model = using_native_modelself.weight = weight- ##################################################- ### Sync Jailbreaking - enhance ##################- ##################################################+ def enhance(+ self,+ attack: str,+ model_callback: CallbackType,+ simulator_model: DeepEvalBaseLLM,+ ) -> str:+ self.simulator_model = simulator_model+ self.model_callback = model_callback- def enhance(self, attack: str) -> str:- """Enhance the attack using tree-based jailbreaking synchronously."""MAX_RUNTIME = 50.0 # Time limit in secondsroot = TreeNode(prompt=attack, score=0, depth=0)start_time = time.time()@@ -170,12 +167,15 @@ class TreeJailbreaking(BaseAttack):)node.children.append(child_node)- ##################################################- ### Async Jailbreaking - a_enhance ###############- ##################################################+ async def a_enhance(+ self,+ attack: str,+ model_callback: CallbackType,+ simulator_model: DeepEvalBaseLLM,+ ) -> str:+ self.model_callback = model_callback+ self.simulator_model = simulator_model- async def a_enhance(self, attack: str, *args, **kwargs) -> str:- """Enhance the attack using tree-based jailbreaking asynchronously."""MAX_RUNTIME = 50.0 # Time limit in secondsroot = TreeNode(prompt=attack, score=0, depth=0)start_time = time.time()diff --git a/deepteam/attacks/single_turn/gray_box/gray_box.py b/deepteam/attacks/single_turn/gray_box/gray_box.pyindex cb35fc87..21d1848d 100644--- a/deepteam/attacks/single_turn/gray_box/gray_box.py+++ b/deepteam/attacks/single_turn/gray_box/gray_box.py@@ -19,33 +19,24 @@ from deepteam.attacks.attack_simulator.utils import (class GrayBox(BaseAttack):- def __init__(- self,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,- weight: int = 1,- ):- self.simulator_model = simulator_model- self.using_native_model = using_native_model+ def __init__(self, weight: int = 1, max_retries: int = 5):self.weight = weight+ self.max_retries = max_retries- ##################################################- ### Sync GrayBox Attack - enhance ################- ##################################################+ def enhance(self, attack: str, simulator_model: DeepEvalBaseLLM) -> str:+ self.simulator_model = simulator_model- def enhance(self, attack: str, max_retries: int = 5) -> str:- """Enhance the attack synchronously with compliance checking and a single progress bar."""prompt = GrayBoxTemplate.enhance(attack)# Progress bar for retries (total count is double the retries: 1 for generation, 1 for compliance check)with tqdm(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Gray Box",unit="step",leave=False,) as pbar:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced attackres: EnhancedAttack = self._generate_schema(prompt, EnhancedAttack@@ -81,24 +72,22 @@ class GrayBox(BaseAttack):# If all retries fail, return the original attackreturn attack- ##################################################- ### Async GrayBox Attack - a_enhance #############- ##################################################-- async def a_enhance(self, attack: str, max_retries: int = 5) -> str:- """Enhance the attack asynchronously with compliance checking and a single progress bar."""+ async def a_enhance(+ self, attack: str, simulator_model: DeepEvalBaseLLM+ ) -> str:+ self.simulator_model = simulator_modelprompt = GrayBoxTemplate.enhance(attack)# Async progress bar for retries (double the count to cover both generation and compliance check)pbar = async_tqdm_bar(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Gray Box",unit="step",leave=False,)try:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced attack asynchronouslyres: EnhancedAttack = await self._a_generate_schema(prompt, EnhancedAttack@@ -143,15 +132,11 @@ class GrayBox(BaseAttack):##################################################def _generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema synchronously."""- return generate_schema(- prompt, schema, self.using_native_model, self.simulator_model- )+ return generate_schema(prompt, schema, False, self.simulator_model)async def _a_generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema asynchronously."""return await a_generate_schema(- prompt, schema, self.using_native_model, self.simulator_model+ prompt, schema, False, self.simulator_model)def get_name(self) -> str:diff --git a/deepteam/attacks/single_turn/math_problem/math_problem.py b/deepteam/attacks/single_turn/math_problem/math_problem.pyindex 8a496c19..04b83b92 100644--- a/deepteam/attacks/single_turn/math_problem/math_problem.py+++ b/deepteam/attacks/single_turn/math_problem/math_problem.py@@ -22,30 +22,28 @@ from deepteam.attacks.attack_simulator.utils import (class MathProblem(BaseAttack):def __init__(self,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,weight: int = 1,+ max_retries: int = 5,):- self.simulator_model = simulator_model- self.using_native_model = using_native_modelself.weight = weight+ self.max_retries = max_retries- ##################################################- ### Sync MathPrompt Attack - enhance #############- ##################################################-- def enhance(self, attack: str, max_retries: int = 5) -> str:- """Enhance the MathPrompt synchronously with compliance checking and a single progress bar."""+ def enhance(+ self,+ attack: str,+ simulator_model: DeepEvalBaseLLM,+ ) -> str:+ self.simulator_model = simulator_modelprompt = MathProblemTemplate.enhance(attack)# Progress bar for retries (total count is double the retries: 1 for generation, 1 for compliance check)with tqdm(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Math Problem Enhancement",unit="step",leave=False,) as pbar:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced promptres: EnhancedAttack = self._generate_schema(prompt, EnhancedAttack@@ -81,24 +79,22 @@ class MathProblem(BaseAttack):# If all retries fail, return the original attackreturn attack- ##################################################- ### Async MathPrompt Attack - a_enhance ##########- ##################################################-- async def a_enhance(self, attack: str, max_retries: int = 5) -> str:- """Enhance the MathPrompt asynchronously with compliance checking and a single progress bar."""+ async def a_enhance(+ self, attack: str, simulator_model: DeepEvalBaseLLM+ ) -> str:+ self.simulator_model = simulator_modelprompt = MathProblemTemplate.enhance(attack)# Async progress bar for retries (double the count to cover both generation and compliance check)pbar = async_tqdm_bar(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Math Problem Enhancement",unit="step",leave=False,)try:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced prompt asynchronouslyres: EnhancedAttack = await self._a_generate_schema(prompt, EnhancedAttack@@ -145,15 +141,11 @@ class MathProblem(BaseAttack):##################################################def _generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema synchronously."""- return generate_schema(- prompt, schema, self.using_native_model, self.simulator_model- )+ return generate_schema(prompt, schema, False, self.simulator_model)async def _a_generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema asynchronously."""return await a_generate_schema(- prompt, schema, self.using_native_model, self.simulator_model+ prompt, schema, False, self.simulator_model)def get_additional_instructions(self) -> str:diff --git a/deepteam/attacks/single_turn/multilingual/multilingual.py b/deepteam/attacks/single_turn/multilingual/multilingual.pyindex 31fec15a..59bfcf92 100644--- a/deepteam/attacks/single_turn/multilingual/multilingual.py+++ b/deepteam/attacks/single_turn/multilingual/multilingual.py@@ -20,33 +20,22 @@ from deepteam.attacks.attack_simulator.utils import (class Multilingual(BaseAttack):- def __init__(- self,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,- weight: int = 1,- ):-- self.simulator_model = simulator_model- self.using_native_model = using_native_model+ def __init__(self, weight: int = 1, max_retries: int = 5):self.weight = weight+ self.max_retries = max_retries- ##################################################- ### Sync Multilingual Attack - enhance ###########- ##################################################-- def enhance(self, attack: str, max_retries: int = 5) -> str:- """Enhance the prompt by translating it to multiple languages synchronously with compliance checking."""+ def enhance(self, attack: str, simulator_model: DeepEvalBaseLLM) -> str:+ self.simulator_model = simulator_modelprompt = MultilingualTemplate.enhance(attack)# Progress bar for retries (total count is double the retries: 1 for generation, 1 for compliance check)with tqdm(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Multilingual Enhancement",unit="step",leave=False,) as pbar:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced promptres: EnhancedAttack = self._generate_schema(prompt, EnhancedAttack@@ -82,24 +71,22 @@ class Multilingual(BaseAttack):# If all retries fail, return the original promptreturn attack- ##################################################- ### Async Multilingual Attack - a_enhance ########- ##################################################-- async def a_enhance(self, attack: str, max_retries: int = 5) -> str:- """Enhance the prompt by translating it to multiple languages asynchronously with compliance checking."""+ async def a_enhance(+ self, attack: str, simulator_model: DeepEvalBaseLLM+ ) -> str:+ self.simulator_model = simulator_modelprompt = MultilingualTemplate.enhance(attack)# Async progress bar for retries (double the count to cover both generation and compliance check)pbar = async_tqdm_bar(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Multilingual Enhancement",unit="step",leave=False,)try:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced prompt asynchronouslyres: EnhancedAttack = await self._a_generate_schema(prompt, EnhancedAttack@@ -146,15 +133,11 @@ class Multilingual(BaseAttack):##################################################def _generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema synchronously."""- return generate_schema(- prompt, schema, self.using_native_model, self.simulator_model- )+ return generate_schema(prompt, schema, False, self.simulator_model)async def _a_generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema asynchronously."""return await a_generate_schema(- prompt, schema, self.using_native_model, self.simulator_model+ prompt, schema, False, self.simulator_model)def get_name(self) -> str:diff --git a/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py b/deepteam/attacks/single_turn/prompt_probing/prompt_probing.pyindex d42b989c..ffc63f53 100644--- a/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py+++ b/deepteam/attacks/single_turn/prompt_probing/prompt_probing.py@@ -21,33 +21,23 @@ from deepteam.attacks.attack_simulator.utils import (class PromptProbing(BaseAttack):- def __init__(- self,- simulator_model: DeepEvalBaseLLM,- using_native_model: bool,- weight: int = 1,- ):- self.simulator_model = simulator_model- self.using_native_model = using_native_model+ def __init__(self, weight: int = 1, max_retries: int = 3):self.weight = weight+ self.max_retries = max_retries- ##################################################- ### Sync PromptProbing - enhance #################- ##################################################-- def enhance(self, attack: str, max_retries: int = 3) -> str:- """Enhance the attack synchronously with compliance checking and a single progress bar."""+ def enhance(self, attack: str, simulator_model: DeepEvalBaseLLM) -> str:+ self.simulator_model = simulator_modelprompt = PromptProbingTemplate.enhance(attack)# Progress bar for retries (total count is double the retries: 1 for generation, 1 for compliance check)with tqdm(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Prompt Probing",unit="step",leave=False,) as pbar:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced attackres: EnhancedAttack = self._generate_schema(prompt, EnhancedAttack@@ -83,24 +73,22 @@ class PromptProbing(BaseAttack):# If all retries fail, return the original attackreturn attack- ##################################################- ### Async PromptProbing Attack - a_enhance #######- ##################################################-- async def a_enhance(self, attack: str, max_retries: int = 3) -> str:- """Enhance the attack asynchronously with compliance checking and a single progress bar."""+ async def a_enhance(+ self, attack: str, simulator_model: DeepEvalBaseLLM+ ) -> str:+ self.simulator_model = simulator_modelprompt = PromptProbingTemplate.enhance(attack)# Async progress bar for retries (double the count to cover both generation and compliance check)pbar = async_tqdm_bar(- total=max_retries * 3,+ total=self.max_retries * 3,desc="...... π Prompt Probing",unit="step",leave=False,)try:- for _ in range(max_retries):+ for _ in range(self.max_retries):# Generate the enhanced attack asynchronouslyres: EnhancedAttack = await self._a_generate_schema(prompt, EnhancedAttack@@ -147,15 +135,11 @@ class PromptProbing(BaseAttack):##################################################def _generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema synchronously."""- return generate_schema(- prompt, schema, self.using_native_model, self.simulator_model- )+ return generate_schema(prompt, schema, False, self.simulator_model)async def _a_generate_schema(self, prompt: str, schema: BaseModel):- """Helper method to generate the schema asynchronously."""return await a_generate_schema(- prompt, schema, self.using_native_model, self.simulator_model+ prompt, schema, False, self.simulator_model)def get_name(self) -> str: