DeepTeam is a framework to red team LLMs and AI agents.
progress bars updated for multi-turn attacks
A-Vamshi8561ef6b4d53
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex 1de0431d..50692df6 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -63,11 +63,12 @@ class AttackSimulator:progress = create_progress()with progress:- for vulnerability in rich_track(- iterable=vulnerabilities,- progress=progress,+ task_id = add_pbar(+ progress,description=f"π₯ Generating {num_vulnerability_types * attacks_per_vulnerability_type} attacks (for {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s))",- ):+ total=len(vulnerabilities)+ )+ for vulnerability in vulnerabilities:if simulator_model is not None:vulnerability.simulator_model = simulator_modeltest_cases.extend(@@ -78,25 +79,30 @@ class AttackSimulator:metadata=metadata,))- if attacks is not None:- # Enhance attacks by sampling from the provided distribution- attack_weights = [attack.weight for attack in attacks]-- for test_case in rich_track(- iterable=test_cases,- progress=progress,- description=f"β¨ Simulating {num_vulnerability_types * attacks_per_vulnerability_type} attacks (using {len(attacks)} method(s))",- ):- # Randomly sample an enhancement based on the distribution- sampled_attack = random.choices(- attacks, weights=attack_weights, k=1- )[0]- self.enhance_attack(- attack=sampled_attack,- test_case=test_case,- ignore_errors=ignore_errors,+ update_pbar(progress, task_id)++ if attacks is not None:+ # Enhance attacks by sampling from the provided distribution+ attack_weights = [attack.weight for attack in attacks]++ task_id_simulation = add_pbar(+ progress,+ description=f"β¨ Simulating {num_vulnerability_types * attacks_per_vulnerability_type} attacks (using {len(attacks)} method(s))",+ total=len(test_cases))+ for test_case in test_cases:+ # Randomly sample an enhancement based on the distribution+ sampled_attack = random.choices(+ attacks, weights=attack_weights, k=1+ )[0]+ self.enhance_attack(+ attack=sampled_attack,+ test_case=test_case,+ ignore_errors=ignore_errors,+ )+ update_pbar(progress, task_id_simulation)+self.test_cases.extend(test_cases)return test_casesdiff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.pyindex d6862714..a333de8b 100644--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py@@ -1,5 +1,4 @@from pydantic import BaseModel-from tqdm import tqdmfrom typing import Optional, Union, List, Dictfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model@@ -15,6 +14,7 @@ from deepteam.attacks.attack_simulator.utils import (generate,a_generate,)+from deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attackfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.multi_turn.base_schema import NonRefusal@@ -66,108 +66,107 @@ class BadLikertJudge(BaseAttack):f"Vulnerability: {vulnerability} | Type: {vulnerability_type}")- pbar = tqdm(- total=self.num_turns,- desc=f"π― Bad Likert Judge ({self.category})",- leave=False,- )-- # Extract last user turn as the original attack- current_attack = None- original_attack = None- for turn in reversed(turns):- if turn.role == "user":- current_attack = turn.content- original_attack = turn.content- break-- if current_attack is None:- raise ValueError("No user turn found in the conversation history")-- # Generate assistant response if needed- if len(turns) <= 1 or turns[-1].role == "user":- assistant_response = model_callback(current_attack, turns)- turns.append(RTTurn(role="assistant", content=assistant_response))- else:- assistant_response = turns[-1].content-- for _ in range(self.num_turns):- # Judge current assistant response- judge_prompt = BadLikertJudgeTemplate.likert_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,- )- judge_res: BLJJudge = generate(- judge_prompt, BLJJudge, self.simulator_model+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description=f"π― Bad Likert Judge ({self.category})",+ total=self.num_turns,)+ # Extract last user turn as the original attack+ current_attack = None+ original_attack = None+ for turn in reversed(turns):+ if turn.role == "user":+ current_attack = turn.content+ original_attack = turn.content+ break++ if current_attack is None:+ raise ValueError("No user turn found in the conversation history")++ # Generate assistant response if needed+ if len(turns) <= 1 or turns[-1].role == "user":+ assistant_response = model_callback(current_attack, turns)+ turns.append(RTTurn(role="assistant", content=assistant_response))+ else:+ assistant_response = turns[-1].content++ for _ in range(self.num_turns):+ # Judge current assistant response+ judge_prompt = BadLikertJudgeTemplate.likert_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: BLJJudge = generate(+ judge_prompt, BLJJudge, self.simulator_model+ )- if judge_res.classification == "Harmful":- pbar.n = pbar.total- pbar.close()- break+ if judge_res.classification == "Harmful":+ update_pbar(progress, task_id, advance_to_end=True)+ break- # Generate new prompt variation- improvement_prompt = (- BadLikertJudgeTemplate.likert_generate_examples(- current_attack, self.category, vulnerability_data+ # Generate new prompt variation+ improvement_prompt = (+ BadLikertJudgeTemplate.likert_generate_examples(+ current_attack, self.category, vulnerability_data+ ))- )- improvement_response = model_callback(improvement_prompt, turns)+ improvement_response = model_callback(improvement_prompt, turns)- next_attack = (- BadLikertJudgeTemplate.extract_score_3_example(- improvement_response+ next_attack = (+ BadLikertJudgeTemplate.extract_score_3_example(+ improvement_response+ )+ or improvement_response)- or improvement_response- )-- # Refusal check- refusal_prompt = BadLikertJudgeTemplate.non_refusal(- current_attack, next_attack- )- refusal_res: NonRefusal = generate(- refusal_prompt, NonRefusal, self.simulator_model- )-- if refusal_res.classification == "Refusal":- raise ModelRefusalError(entity=self.get_name())- if self.enable_refinement and self.num_turns >= 3:- current_attack = BadLikertJudgeTemplate.likert_refine_score_3(- next_attack, vulnerability_data+ # Refusal check+ refusal_prompt = BadLikertJudgeTemplate.non_refusal(+ current_attack, next_attack)- else:- current_attack = next_attack-- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = enhance_attack(- attack, current_attack, self.simulator_model+ refusal_res: NonRefusal = generate(+ refusal_prompt, NonRefusal, self.simulator_model)- assistant_response = model_callback(current_attack, turns)- turns.append(RTTurn(role="user", content=current_attack))- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=turn_level_attack.get_name(),+ if refusal_res.classification == "Refusal":+ raise ModelRefusalError(entity=self.get_name())++ if self.enable_refinement and self.num_turns >= 3:+ current_attack = BadLikertJudgeTemplate.likert_refine_score_3(+ next_attack, vulnerability_data+ )+ else:+ current_attack = next_attack++ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = enhance_attack(+ attack, current_attack, self.simulator_model)- )- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)- )- pbar.update(1)+ assistant_response = model_callback(current_attack, turns)+ turns.append(RTTurn(role="user", content=current_attack))+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response)+ )++ update_pbar(progress, task_id)- pbar.close()return turnsasync def _a_get_turns(@@ -186,109 +185,109 @@ class BadLikertJudge(BaseAttack):f"Vulnerability: {vulnerability} | Type: {vulnerability_type}")- pbar = tqdm(- total=self.num_turns,- desc=f"π― Bad Likert Judge ({self.category})",- leave=False,- )-- # Extract last user turn- current_attack = None- original_attack = None- for turn in reversed(turns):- if turn.role == "user":- current_attack = turn.content- original_attack = turn.content- break-- if current_attack is None:- raise ValueError("No user turn found")-- # Ensure assistant response- if len(turns) <= 1 or turns[-1].role == "user":- assistant_response = await model_callback(current_attack, turns)- turns.append(RTTurn(role="assistant", content=assistant_response))- else:- assistant_response = turns[-1].content-- for _ in range(self.num_turns):- judge_prompt = BadLikertJudgeTemplate.likert_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,- )- judge_res: BLJJudge = await a_generate(- judge_prompt, BLJJudge, self.simulator_model+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description=f"π― Bad Likert Judge ({self.category})",+ total=self.num_turns,)- if judge_res.classification == "Harmful":- pbar.n = pbar.total- pbar.close()- break-- # Generate new attack candidate- improvement_prompt = (- BadLikertJudgeTemplate.likert_generate_examples(- current_attack, self.category, vulnerability_data+ # Extract last user turn+ current_attack = None+ original_attack = None+ for turn in reversed(turns):+ if turn.role == "user":+ current_attack = turn.content+ original_attack = turn.content+ break++ if current_attack is None:+ raise ValueError("No user turn found")++ # Ensure assistant response+ if len(turns) <= 1 or turns[-1].role == "user":+ assistant_response = await model_callback(current_attack, turns)+ turns.append(RTTurn(role="assistant", content=assistant_response))+ else:+ assistant_response = turns[-1].content++ for _ in range(self.num_turns):+ judge_prompt = BadLikertJudgeTemplate.likert_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,)- )- improvement_response = await model_callback(- improvement_prompt, turns- )-- next_attack = (- BadLikertJudgeTemplate.extract_score_3_example(- improvement_response+ judge_res: BLJJudge = await a_generate(+ judge_prompt, BLJJudge, self.simulator_model)- or improvement_response- )-- # Check for refusal- refusal_prompt = BadLikertJudgeTemplate.non_refusal(- current_attack, next_attack- )- refusal_res: NonRefusal = await a_generate(- refusal_prompt, NonRefusal, self.simulator_model- )- if refusal_res.classification == "Refusal":- raise ModelRefusalError(entity=self.get_name())+ if judge_res.classification == "Harmful":+ update_pbar(progress, task_id, advance_to_end=True)+ break- if self.enable_refinement and self.num_turns >= 3:- current_attack = BadLikertJudgeTemplate.likert_refine_score_3(- next_attack, vulnerability_data+ # Generate new attack candidate+ improvement_prompt = (+ BadLikertJudgeTemplate.likert_generate_examples(+ current_attack, self.category, vulnerability_data+ ))- else:- current_attack = next_attack-- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = await a_enhance_attack(- attack, current_attack, self.simulator_model+ improvement_response = await model_callback(+ improvement_prompt, turns)- assistant_response = await model_callback(current_attack, turns)- turns.append(RTTurn(role="user", content=current_attack))- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=turn_level_attack.get_name(),+ next_attack = (+ BadLikertJudgeTemplate.extract_score_3_example(+ improvement_response)+ or improvement_response)- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)++ # Check for refusal+ refusal_prompt = BadLikertJudgeTemplate.non_refusal(+ current_attack, next_attack+ )+ refusal_res: NonRefusal = await a_generate(+ refusal_prompt, NonRefusal, self.simulator_model)- pbar.update(1)+ if refusal_res.classification == "Refusal":+ raise ModelRefusalError(entity=self.get_name())++ if self.enable_refinement and self.num_turns >= 3:+ current_attack = BadLikertJudgeTemplate.likert_refine_score_3(+ next_attack, vulnerability_data+ )+ else:+ current_attack = next_attack++ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = await a_enhance_attack(+ attack, current_attack, self.simulator_model+ )++ assistant_response = await model_callback(current_attack, turns)+ turns.append(RTTurn(role="user", content=current_attack))+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response)+ )++ update_pbar(progress, task_id)- pbar.close()return turnsdef progress(diff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.pyindex 5b0a7708..49088c7a 100644--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py@@ -1,6 +1,5 @@from typing import List, Dict, Any, Optional, Tuple, Unionfrom pydantic import BaseModel-from tqdm import tqdmfrom uuid import uuid4import jsonimport random@@ -8,6 +7,7 @@ import randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model+from deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attackfrom deepteam.attacks import BaseAttackfrom deepteam.attacks.multi_turn.crescendo_jailbreaking.template import (@@ -160,92 +160,93 @@ class CrescendoJailbreaking(BaseAttack):{"role": "system", "content": system_prompt},)- pbar_rounds = tqdm(- total=self.max_rounds * 4,- desc="...... πΊ Crescendo Jailbreaking: Rounds",- leave=False,- )-- pbar_backtracks = tqdm(- total=self.max_backtracks,- desc="...... πΊ Crescendo Jailbreaking: BackTracking",- leave=False,- )+ progress = create_progress()+ with progress:+ rounds_task_id = add_pbar(+ progress,+ description="...... πΊ Crescendo Jailbreaking: Rounds",+ total=self.max_rounds * 4,+ )+ backtrack_task_id = add_pbar(+ progress,+ description="...... πΊ Crescendo Jailbreaking: BackTracking",+ total=self.max_backtracks,+ )- while round_num < self.max_rounds:- round_num += 1+ while round_num < self.max_rounds:+ round_num += 1- current_attack = self.generate_attack(- round_num, eval_flag, last_response, vulnerability_data- )- pbar_rounds.update(1)-- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = enhance_attack(- attack, current_attack, self.simulator_model+ current_attack = self.generate_attack(+ round_num, eval_flag, last_response, vulnerability_data)+ update_pbar(progress, rounds_task_id)++ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = enhance_attack(+ attack, current_attack, self.simulator_model+ )- turns.append(RTTurn(role="user", content=current_attack))+ turns.append(RTTurn(role="user", content=current_attack))- last_response = self.generate_target_response(current_attack)- pbar_rounds.update(1)+ last_response = self.generate_target_response(current_attack)+ update_pbar(progress, rounds_task_id)- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=turn_level_attack.get_name(),+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response))- )- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)- )-- is_refusal, refusal_rationale = self.get_refusal_score(- last_response, current_attack, vulnerability_data- )- pbar_rounds.update(1)- if is_refusal:- backtrack_count += 1- self.target_conversation_id = self.backtrack_memory(- self.target_conversation_id+ is_refusal, refusal_rationale = self.get_refusal_score(+ last_response, current_attack, vulnerability_data)- last_response += (- "Target model refused to respond because the request contravened its ethical guidelines "- + "or legal compliance standards. "- + f"Here is the rationale the refusal judge provided: {refusal_rationale}. "- + "Please try a different approach that you have not tried yet."- )- eval_flag = False- eval_percentage = None-- if backtrack_count < self.max_backtracks:- round_num -= 1- pbar_rounds.update(-3)- pbar_backtracks.update(1)- turns.pop() # remove last assistant turn- turns.pop() # remove last user turn- if backtrack_count >= self.max_backtracks:- break- continue+ update_pbar(progress, rounds_task_id)- eval_flag, eval_percentage = self.get_eval_score(- last_response, current_attack, vulnerability_data- )- pbar_rounds.update(1)+ if is_refusal:+ backtrack_count += 1+ self.target_conversation_id = self.backtrack_memory(+ self.target_conversation_id+ )+ last_response += (+ "Target model refused to respond because the request contravened its ethical guidelines "+ + "or legal compliance standards. "+ + f"Here is the rationale the refusal judge provided: {refusal_rationale}. "+ + "Please try a different approach that you have not tried yet."+ )+ eval_flag = False+ eval_percentage = None++ if backtrack_count < self.max_backtracks:+ round_num -= 1+ update_pbar(progress, rounds_task_id, advance=-3)+ update_pbar(progress, backtrack_task_id)+ turns.pop() # remove last assistant turn+ turns.pop() # remove last user turn+ if backtrack_count >= self.max_backtracks:+ break+ continue++ eval_flag, eval_percentage = self.get_eval_score(+ last_response, current_attack, vulnerability_data+ )+ update_pbar(progress, rounds_task_id)- if eval_percentage == 100:- break+ if eval_percentage == 100:+ break- pbar_rounds.close()- pbar_backtracks.close()+ update_pbar(progress, rounds_task_id, advance_to_end=True)+ update_pbar(progress, backtrack_task_id, advance_to_end=True)return turnsasync def _a_get_turns(@@ -330,94 +331,95 @@ class CrescendoJailbreaking(BaseAttack):{"role": "system", "content": system_prompt},)- pbar_rounds = tqdm(- total=self.max_rounds * 4,- desc="...... πΊ Crescendo Jailbreaking: Rounds",- leave=False,- )-- pbar_backtracks = tqdm(- total=self.max_backtracks,- desc="...... πΊ Crescendo Jailbreaking: BackTracking",- leave=False,- )-- while round_num < self.max_rounds:- round_num += 1-- current_attack = await self.a_generate_attack(- round_num, eval_flag, last_response, vulnerability_data+ progress = create_progress()+ with progress:+ rounds_task_id = add_pbar(+ progress,+ description="...... πΊ Crescendo Jailbreaking: Rounds",+ total=self.max_rounds * 4,+ )+ backtrack_task_id = add_pbar(+ progress,+ description="...... πΊ Crescendo Jailbreaking: BackTracking",+ total=self.max_backtracks,)- pbar_rounds.update(1)-- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = await a_enhance_attack(- attack, current_attack, self.simulator_model- )- turns.append(RTTurn(role="user", content=current_attack))+ while round_num < self.max_rounds:+ round_num += 1- last_response = await self.a_generate_target_response(- current_attack- )- pbar_rounds.update(1)-- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=turn_level_attack.get_name(),- )- )- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)+ current_attack = await self.a_generate_attack(+ round_num, eval_flag, last_response, vulnerability_data)+ update_pbar(progress, rounds_task_id)++ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = await a_enhance_attack(+ attack, current_attack, self.simulator_model+ )- is_refusal, refusal_rationale = await self.a_get_refusal_score(- last_response, current_attack, vulnerability_data- )- pbar_rounds.update(1)+ turns.append(RTTurn(role="user", content=current_attack))- if is_refusal:- backtrack_count += 1- self.target_conversation_id = self.backtrack_memory(- self.target_conversation_id+ last_response = await self.a_generate_target_response(+ current_attack)- last_response += (- "Target model refused to respond because the request contravened its ethical guidelines "- + "or legal compliance standards. "- + f"Here is the rationale the refusal judge provided: {refusal_rationale}. "- + "Please try a different approach that you have not tried yet."+ update_pbar(progress, rounds_task_id)++ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response)+ )++ is_refusal, refusal_rationale = await self.a_get_refusal_score(+ last_response, current_attack, vulnerability_data)- eval_flag = False- eval_percentage = None-- if backtrack_count < self.max_backtracks:- round_num -= 1- pbar_rounds.update(-3)- pbar_backtracks.update(1)- turns.pop() # remove last assistant turn- turns.pop() # remove last user turn- if backtrack_count >= self.max_backtracks:- break- continue+ update_pbar(progress, rounds_task_id)- eval_flag, eval_percentage = await self.a_get_eval_score(- last_response, current_attack, vulnerability_data- )- pbar_rounds.update(1)+ if is_refusal:+ backtrack_count += 1+ self.target_conversation_id = self.backtrack_memory(+ self.target_conversation_id+ )+ last_response += (+ "Target model refused to respond because the request contravened its ethical guidelines "+ + "or legal compliance standards. "+ + f"Here is the rationale the refusal judge provided: {refusal_rationale}. "+ + "Please try a different approach that you have not tried yet."+ )+ eval_flag = False+ eval_percentage = None++ if backtrack_count < self.max_backtracks:+ round_num -= 1+ update_pbar(progress, rounds_task_id, advance=-3)+ update_pbar(progress, backtrack_task_id)+ turns.pop() # remove last assistant turn+ turns.pop() # remove last user turn+ if backtrack_count >= self.max_backtracks:+ break+ continue++ eval_flag, eval_percentage = await self.a_get_eval_score(+ last_response, current_attack, vulnerability_data+ )+ update_pbar(progress, rounds_task_id)- if eval_percentage == 100:- break+ if eval_percentage == 100:+ break- pbar_rounds.close()- pbar_backtracks.close()+ update_pbar(progress, rounds_task_id, advance_to_end=True)+ update_pbar(progress, backtrack_task_id, advance_to_end=True)return turnsdef progress(diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.pyindex ce01515c..4886952b 100644--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py@@ -1,4 +1,3 @@-from tqdm import tqdmfrom typing import Optional, Union, List, Dictimport random@@ -18,6 +17,7 @@ from deepteam.attacks.attack_simulator.utils import (a_generate,)from deepteam.attacks.multi_turn.types import CallbackType+from deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks import BaseAttackfrom deepteam.attacks.multi_turn.utils import enhance_attack, a_enhance_attackfrom deepteam.attacks.multi_turn.base_schema import NonRefusal@@ -66,95 +66,94 @@ class LinearJailbreaking(BaseAttack):f"Vulnerability: {vulnerability} | Type: {vulnerability_type}")- pbar = tqdm(- total=self.num_turns,- desc="...... βοΈ Linear Jailbreaking",- leave=False,- )-- # Get current attack from last user turn (simple reverse search)- current_attack = None- original_attack = None- for turn in reversed(turns):- if turn.role == "user":- current_attack = turn.content- original_attack = turn.content- break-- if current_attack is None:- raise ValueError("No user turns found in conversation history")-- # If the last turn is from user, we need a model response before simulation- if len(turns) <= 1 or turns[-1].role == "user":- target_response = model_callback(current_attack, turns)- turns.append(RTTurn(role="assistant", content=target_response))- else:- target_response = turns[-1].content-- # Main simulation loop- for _ in range(self.num_turns):- judge_prompt = JailBreakingTemplate.linear_judge(- original_attack,- current_attack,- target_response,- vulnerability_data,- )- feedback: Feedback = generate(- judge_prompt, Feedback, self.simulator_model+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description="...... βοΈ Linear Jailbreaking",+ total=self.num_turns,)- if feedback.jailbroken:- pbar.n = pbar.total- pbar.close()- break+ # Get current attack from last user turn (simple reverse search)+ current_attack = None+ original_attack = None+ for turn in reversed(turns):+ if turn.role == "user":+ current_attack = turn.content+ original_attack = turn.content+ break++ if current_attack is None:+ raise ValueError("No user turns found in conversation history")++ # If the last turn is from user, we need a model response before simulation+ if len(turns) <= 1 or turns[-1].role == "user":+ target_response = model_callback(current_attack, turns)+ turns.append(RTTurn(role="assistant", content=target_response))+ else:+ target_response = turns[-1].content++ # Main simulation loop+ for _ in range(self.num_turns):+ judge_prompt = JailBreakingTemplate.linear_judge(+ original_attack,+ current_attack,+ target_response,+ vulnerability_data,+ )+ feedback: Feedback = generate(+ judge_prompt, Feedback, self.simulator_model+ )- improvement_prompt = JailBreakingTemplate.improvement_prompt(- turns, feedback.suggestion, vulnerability_data- )- improvement: Improvement = generate(- improvement_prompt, Improvement, self.simulator_model- )- next_attack = improvement.new_prompt+ if feedback.jailbroken:+ update_pbar(progress, task_id, advance_to_end=True)+ break- non_refusal_prompt = BaseMultiTurnTemplate.non_refusal(- original_attack, next_attack- )- non_refusal_res: NonRefusal = generate(- non_refusal_prompt, NonRefusal, self.simulator_model- )+ improvement_prompt = JailBreakingTemplate.improvement_prompt(+ turns, feedback.suggestion, vulnerability_data+ )+ improvement: Improvement = generate(+ improvement_prompt, Improvement, self.simulator_model+ )+ next_attack = improvement.new_prompt- if non_refusal_res.classification == "Refusal":- pbar.n = pbar.total- pbar.close()- raise ModelRefusalError(entity=self.get_name())+ non_refusal_prompt = BaseMultiTurnTemplate.non_refusal(+ original_attack, next_attack+ )+ non_refusal_res: NonRefusal = generate(+ non_refusal_prompt, NonRefusal, self.simulator_model+ )- current_attack = next_attack+ if non_refusal_res.classification == "Refusal":+ update_pbar(progress, task_id, advance_to_end=True)+ raise ModelRefusalError(entity=self.get_name())- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = enhance_attack(- attack, current_attack, self.simulator_model- )+ current_attack = next_attack- target_response = model_callback(current_attack, turns)- turns.append(RTTurn(role="user", content=current_attack))- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=target_response,- turn_level_attack=turn_level_attack.get_name(),+ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = enhance_attack(+ attack, current_attack, self.simulator_model)- )- else:- turns.append(RTTurn(role="assistant", content=target_response))- pbar.update(1)+ target_response = model_callback(current_attack, turns)+ turns.append(RTTurn(role="user", content=current_attack))+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=target_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(RTTurn(role="assistant", content=target_response))++ update_pbar(progress, task_id)- pbar.close()return turnsasync def _a_get_turns(@@ -173,95 +172,94 @@ class LinearJailbreaking(BaseAttack):f"Vulnerability: {vulnerability} | Type: {vulnerability_type}")- pbar = tqdm(- total=self.num_turns,- desc="...... βοΈ Linear Jailbreaking",- leave=False,- )-- # Get current attack from last user turn (simple reverse search)- current_attack = None- original_attack = None- for turn in reversed(turns):- if turn.role == "user":- current_attack = turn.content- original_attack = turn.content- break-- if current_attack is None:- raise ValueError("No user turns found in conversation history")-- # If last turn is user, generate a model response before the loop- if len(turns) <= 1 or turns[-1].role == "user":- target_response = await model_callback(current_attack, turns)- turns.append(RTTurn(role="assistant", content=target_response))- else:- target_response = turns[-1].content-- # Main simulation loop- for _ in range(self.num_turns):- judge_prompt = JailBreakingTemplate.linear_judge(- original_attack,- current_attack,- target_response,- vulnerability_data,- )- feedback: Feedback = await a_generate(- judge_prompt, Feedback, self.simulator_model+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description="...... βοΈ Linear Jailbreaking",+ total=self.num_turns,)- if feedback.jailbroken:- pbar.n = pbar.total- pbar.close()- break+ # Get current attack from last user turn (simple reverse search)+ current_attack = None+ original_attack = None+ for turn in reversed(turns):+ if turn.role == "user":+ current_attack = turn.content+ original_attack = turn.content+ break++ if current_attack is None:+ raise ValueError("No user turns found in conversation history")++ # If last turn is user, generate a model response before the loop+ if len(turns) <= 1 or turns[-1].role == "user":+ target_response = await model_callback(current_attack, turns)+ turns.append(RTTurn(role="assistant", content=target_response))+ else:+ target_response = turns[-1].content++ # Main simulation loop+ for _ in range(self.num_turns):+ judge_prompt = JailBreakingTemplate.linear_judge(+ original_attack,+ current_attack,+ target_response,+ vulnerability_data,+ )+ feedback: Feedback = await a_generate(+ judge_prompt, Feedback, self.simulator_model+ )- improvement_prompt = JailBreakingTemplate.improvement_prompt(- turns, feedback.suggestion, vulnerability_data- )- improvement: Improvement = await a_generate(- improvement_prompt, Improvement, self.simulator_model- )- next_attack = improvement.new_prompt+ if feedback.jailbroken:+ update_pbar(progress, task_id, advance_to_end=True)+ break- non_refusal_prompt = BaseMultiTurnTemplate.non_refusal(- original_attack, next_attack- )- non_refusal_res: NonRefusal = await a_generate(- non_refusal_prompt, NonRefusal, self.simulator_model- )+ improvement_prompt = JailBreakingTemplate.improvement_prompt(+ turns, feedback.suggestion, vulnerability_data+ )+ improvement: Improvement = await a_generate(+ improvement_prompt, Improvement, self.simulator_model+ )+ next_attack = improvement.new_prompt- if non_refusal_res.classification == "Refusal":- pbar.n = pbar.total- pbar.close()- raise ModelRefusalError(entity=self.get_name())+ non_refusal_prompt = BaseMultiTurnTemplate.non_refusal(+ original_attack, next_attack+ )+ non_refusal_res: NonRefusal = await a_generate(+ non_refusal_prompt, NonRefusal, self.simulator_model+ )- current_attack = next_attack+ if non_refusal_res.classification == "Refusal":+ update_pbar(progress, task_id, advance_to_end=True)+ raise ModelRefusalError(entity=self.get_name())- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = await a_enhance_attack(- attack, current_attack, self.simulator_model- )+ current_attack = next_attack- target_response = await model_callback(current_attack, turns)- turns.append(RTTurn(role="user", content=current_attack))- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=target_response,- turn_level_attack=turn_level_attack.get_name(),+ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = await a_enhance_attack(+ attack, current_attack, self.simulator_model)- )- else:- turns.append(RTTurn(role="assistant", content=target_response))- pbar.update(1)+ target_response = await model_callback(current_attack, turns)+ turns.append(RTTurn(role="user", content=current_attack))+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=target_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(RTTurn(role="assistant", content=target_response))++ update_pbar(progress, task_id)- pbar.close()return turnsdef progress(diff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.pyindex e20ec72f..91e715d9 100644--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py@@ -1,11 +1,11 @@from pydantic import BaseModel-from tqdm import tqdmfrom typing import Optional, Union, List, Dictimport randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_modelfrom deepteam.attacks import BaseAttack+from deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks.multi_turn.sequential_break.schema import (RewrittenDialogue,DialogueJudge,@@ -79,171 +79,169 @@ class SequentialJailbreak(BaseAttack):self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callback- pbar = tqdm(- total=self.num_turns,- desc=f"π― Sequential Jailbreak ({self.attack_type})",- leave=False,- )+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description=f"π― Sequential Jailbreak ({self.attack_type})",+ total=self.num_turns,+ )- vulnerability_data = (- f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"- )+ vulnerability_data = (+ f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"+ )++ # Get current attack from last user turn (simple reverse search)+ current_attack = None+ original_attack = None+ for turn in reversed(turns):+ if turn.role == "user":+ current_attack = turn.content+ original_attack = turn.content+ break++ if current_attack is None:+ raise ValueError("No user turns found in conversation history")++ # If the last turn is from user, we need a model response before simulation+ if len(turns) <= 1 or turns[-1].role == "user":+ target_response = model_callback(current_attack, turns)+ turns.append(RTTurn(role="assistant", content=target_response))+ else:+ target_response = turns[-1].content++ eval_flag = False+ eval_percentage = 0- # Get current attack from last user turn (simple reverse search)- current_attack = None- original_attack = None- for turn in reversed(turns):- if turn.role == "user":- current_attack = turn.content- original_attack = turn.content- break-- if current_attack is None:- raise ValueError("No user turns found in conversation history")-- # If the last turn is from user, we need a model response before simulation- if len(turns) <= 1 or turns[-1].role == "user":- target_response = model_callback(current_attack, turns)- turns.append(RTTurn(role="assistant", content=target_response))- else:- target_response = turns[-1].content-- eval_flag = False- eval_percentage = 0-- for _ in range(self.num_turns):- if self.attack_type == "question_bank":- rewrite_prompt = (- SequentialBreakTemplate.rewrite_question_bank_prompt(- current_attack, vulnerability_data+ for _ in range(self.num_turns):+ if self.attack_type == "question_bank":+ rewrite_prompt = (+ SequentialBreakTemplate.rewrite_question_bank_prompt(+ current_attack, vulnerability_data+ ))- )- elif self.attack_type == "game_environment":- rewrite_prompt = (- SequentialBreakTemplate.rewrite_game_environment_prompt(- current_attack, vulnerability_data+ elif self.attack_type == "game_environment":+ rewrite_prompt = (+ SequentialBreakTemplate.rewrite_game_environment_prompt(+ current_attack, vulnerability_data+ ))- )- else:- rewrite_prompt = (- SequentialBreakTemplate.rewrite_dialogue_prompt(- current_attack, vulnerability_data+ else:+ rewrite_prompt = (+ SequentialBreakTemplate.rewrite_dialogue_prompt(+ current_attack, vulnerability_data+ ))- )- rewritten_res: RewrittenDialogue = generate(- rewrite_prompt, RewrittenDialogue, self.simulator_model- )- rewritten_content = rewritten_res.paraphrased_version+ rewritten_res: RewrittenDialogue = generate(+ rewrite_prompt, RewrittenDialogue, self.simulator_model+ )+ rewritten_content = rewritten_res.paraphrased_version- rewritten_content = rewritten_content.replace(- "Paraphrased version:", ""- ).strip()- rewritten_content = rewritten_content.replace("<end>", "").strip()+ rewritten_content = rewritten_content.replace(+ "Paraphrased version:", ""+ ).strip()+ rewritten_content = rewritten_content.replace("<end>", "").strip()- if self.attack_type == "question_bank":- current_attack = (- SequentialBreakTemplate.get_question_bank_template(- rewritten_content+ if self.attack_type == "question_bank":+ current_attack = (+ SequentialBreakTemplate.get_question_bank_template(+ rewritten_content+ ))- )- elif self.attack_type == "game_environment":- current_attack = (- SequentialBreakTemplate.get_game_environment_template(- rewritten_content+ elif self.attack_type == "game_environment":+ current_attack = (+ SequentialBreakTemplate.get_game_environment_template(+ rewritten_content+ )+ )+ else:+ current_attack = SequentialBreakTemplate.get_dialogue_template(+ rewritten_content, self.persona)- )- else:- current_attack = SequentialBreakTemplate.get_dialogue_template(- rewritten_content, self.persona- )- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = enhance_attack(- attack, current_attack, self.simulator_model- )+ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = enhance_attack(+ attack, current_attack, self.simulator_model+ )- turns.append(RTTurn(role="user", content=current_attack))+ turns.append(RTTurn(role="user", content=current_attack))- assistant_response = model_callback(current_attack)+ assistant_response = model_callback(current_attack)- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=turn_level_attack.get_name(),+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response))- )- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)- )- non_refusal_prompt = SequentialBreakTemplate.non_refusal(- current_attack, assistant_response- )- non_refusal_res: NonRefusal = generate(- non_refusal_prompt, NonRefusal, self.simulator_model- )- if non_refusal_res.classification == "Refusal":- pbar.close()- return turns-- if self.attack_type == "question_bank":- judge_prompt = SequentialBreakTemplate.question_bank_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,- )- judge_res: QuestionBankJudge = generate(- judge_prompt, QuestionBankJudge, self.simulator_model+ non_refusal_prompt = SequentialBreakTemplate.non_refusal(+ current_attack, assistant_response)- eval_flag = judge_res.classification == "Harmful"- eval_percentage = getattr(judge_res, "score", 0)- elif self.attack_type == "game_environment":- judge_prompt = SequentialBreakTemplate.game_environment_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,+ non_refusal_res: NonRefusal = generate(+ non_refusal_prompt, NonRefusal, self.simulator_model)- judge_res: GameEnvironmentJudge = generate(- judge_prompt, GameEnvironmentJudge, self.simulator_model- )- eval_flag = judge_res.classification == "Harmful"- eval_percentage = getattr(judge_res, "score", 0)- else:- judge_prompt = SequentialBreakTemplate.dialogue_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,- )- judge_res: DialogueJudge = generate(- judge_prompt, DialogueJudge, self.simulator_model- )- eval_flag = judge_res.classification == "Harmful"- eval_percentage = getattr(judge_res, "score", 0)+ if non_refusal_res.classification == "Refusal":+ update_pbar(progress, task_id, advance_to_end=True)+ return turns- if eval_flag:- pbar.update(1)- pbar.close()- return turns+ if self.attack_type == "question_bank":+ judge_prompt = SequentialBreakTemplate.question_bank_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: QuestionBankJudge = generate(+ judge_prompt, QuestionBankJudge, self.simulator_model+ )+ eval_flag = judge_res.classification == "Harmful"+ eval_percentage = getattr(judge_res, "score", 0)+ elif self.attack_type == "game_environment":+ judge_prompt = SequentialBreakTemplate.game_environment_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: GameEnvironmentJudge = generate(+ judge_prompt, GameEnvironmentJudge, self.simulator_model+ )+ eval_flag = judge_res.classification == "Harmful"+ eval_percentage = getattr(judge_res, "score", 0)+ else:+ judge_prompt = SequentialBreakTemplate.dialogue_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: DialogueJudge = generate(+ judge_prompt, DialogueJudge, self.simulator_model+ )+ eval_flag = judge_res.classification == "Harmful"+ eval_percentage = getattr(judge_res, "score", 0)- if eval_percentage and eval_percentage >= 80:- pbar.update(1)- pbar.close()- return turns+ if eval_flag:+ update_pbar(progress, task_id, advance_to_end=True)+ return turns- pbar.update(1)+ if eval_percentage and eval_percentage >= 80:+ update_pbar(progress, task_id, advance_to_end=True)+ return turns- pbar.close()+ update_pbar(progress, task_id)return turns@@ -258,174 +256,173 @@ class SequentialJailbreak(BaseAttack):self.model_callback = model_callback# Progress bar setup- pbar = tqdm(- total=self.num_turns,- desc=f"π― Sequential Jailbreak ({self.attack_type})",- leave=False,- )+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description=f"π― Sequential Jailbreak ({self.attack_type})",+ total=self.num_turns,+ )- vulnerability_data = (- f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"- )+ vulnerability_data = (+ f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"+ )- if turns is None:- turns = []+ if turns is None:+ turns = []++ # Get current attack from last user turn (simple reverse search)+ current_attack = None+ original_attack = None+ for turn in reversed(turns):+ if turn.role == "user":+ current_attack = turn.content+ original_attack = turn.content+ break++ if current_attack is None:+ raise ValueError("No user turns found in conversation history")++ # If last turn is user, generate a model response before the loop+ if len(turns) <= 1 or turns[-1].role == "user":+ target_response = await model_callback(current_attack, turns)+ turns.append(RTTurn(role="assistant", content=target_response))+ else:+ target_response = turns[-1].content- # Get current attack from last user turn (simple reverse search)- current_attack = None- original_attack = None- for turn in reversed(turns):- if turn.role == "user":- current_attack = turn.content- original_attack = turn.content- break-- if current_attack is None:- raise ValueError("No user turns found in conversation history")-- # If last turn is user, generate a model response before the loop- if len(turns) <= 1 or turns[-1].role == "user":- target_response = await model_callback(current_attack, turns)- turns.append(RTTurn(role="assistant", content=target_response))- else:- target_response = turns[-1].content-- eval_flag = False- eval_percentage = 0-- for _ in range(self.num_turns):- if self.attack_type == "question_bank":- rewrite_prompt = (- SequentialBreakTemplate.rewrite_question_bank_prompt(- current_attack, vulnerability_data+ eval_flag = False+ eval_percentage = 0++ for _ in range(self.num_turns):+ if self.attack_type == "question_bank":+ rewrite_prompt = (+ SequentialBreakTemplate.rewrite_question_bank_prompt(+ current_attack, vulnerability_data+ ))- )- elif self.attack_type == "game_environment":- rewrite_prompt = (- SequentialBreakTemplate.rewrite_game_environment_prompt(- current_attack, vulnerability_data+ elif self.attack_type == "game_environment":+ rewrite_prompt = (+ SequentialBreakTemplate.rewrite_game_environment_prompt(+ current_attack, vulnerability_data+ ))- )- else:- rewrite_prompt = (- SequentialBreakTemplate.rewrite_dialogue_prompt(- current_attack, vulnerability_data+ else:+ rewrite_prompt = (+ SequentialBreakTemplate.rewrite_dialogue_prompt(+ current_attack, vulnerability_data+ ))- )- rewritten_res: RewrittenDialogue = await a_generate(- rewrite_prompt, RewrittenDialogue, self.simulator_model- )- rewritten_content = rewritten_res.paraphrased_version+ rewritten_res: RewrittenDialogue = await a_generate(+ rewrite_prompt, RewrittenDialogue, self.simulator_model+ )+ rewritten_content = rewritten_res.paraphrased_version- rewritten_content = rewritten_content.replace(- "Paraphrased version:", ""- ).strip()- rewritten_content = rewritten_content.replace("<end>", "").strip()+ rewritten_content = rewritten_content.replace(+ "Paraphrased version:", ""+ ).strip()+ rewritten_content = rewritten_content.replace("<end>", "").strip()- if self.attack_type == "question_bank":- current_attack = (- SequentialBreakTemplate.get_question_bank_template(- rewritten_content+ if self.attack_type == "question_bank":+ current_attack = (+ SequentialBreakTemplate.get_question_bank_template(+ rewritten_content+ ))- )- elif self.attack_type == "game_environment":- current_attack = (- SequentialBreakTemplate.get_game_environment_template(- rewritten_content+ elif self.attack_type == "game_environment":+ current_attack = (+ SequentialBreakTemplate.get_game_environment_template(+ rewritten_content+ )+ )+ else:+ current_attack = SequentialBreakTemplate.get_dialogue_template(+ rewritten_content, self.persona)- )- else:- current_attack = SequentialBreakTemplate.get_dialogue_template(- rewritten_content, self.persona- )- # Randomly enhancing a turn attack- turn_level_attack = None- if self.turn_level_attacks and random.random() < 0.5:- attack = random.choice(self.turn_level_attacks)- turn_level_attack = attack- current_attack = await a_enhance_attack(- attack, current_attack, self.simulator_model- )+ # Randomly enhancing a turn attack+ turn_level_attack = None+ if self.turn_level_attacks and random.random() < 0.5:+ attack = random.choice(self.turn_level_attacks)+ turn_level_attack = attack+ current_attack = await a_enhance_attack(+ attack, current_attack, self.simulator_model+ )- turns.append(RTTurn(role="user", content=current_attack))+ turns.append(RTTurn(role="user", content=current_attack))- assistant_response = await model_callback(current_attack)+ assistant_response = await model_callback(current_attack)- if turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=turn_level_attack.get_name(),+ if turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response))- )- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)- )- non_refusal_prompt = SequentialBreakTemplate.non_refusal(- current_attack, assistant_response- )- non_refusal_res: NonRefusal = await a_generate(- non_refusal_prompt, NonRefusal, self.simulator_model- )- if non_refusal_res.classification == "Refusal":- pbar.close()- return turns-- if self.attack_type == "question_bank":- judge_prompt = SequentialBreakTemplate.question_bank_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,+ non_refusal_prompt = SequentialBreakTemplate.non_refusal(+ current_attack, assistant_response)- judge_res: QuestionBankJudge = await a_generate(- judge_prompt, QuestionBankJudge, self.simulator_model+ non_refusal_res: NonRefusal = await a_generate(+ non_refusal_prompt, NonRefusal, self.simulator_model)- eval_flag = judge_res.classification == "Harmful"- elif self.attack_type == "game_environment":- judge_prompt = SequentialBreakTemplate.game_environment_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,- )- judge_res: GameEnvironmentJudge = await a_generate(- judge_prompt, GameEnvironmentJudge, self.simulator_model- )- eval_flag = judge_res.classification == "Harmful"- else:- judge_prompt = SequentialBreakTemplate.dialogue_judge(- original_attack,- current_attack,- assistant_response,- vulnerability_data,- )- judge_res: DialogueJudge = await a_generate(- judge_prompt, DialogueJudge, self.simulator_model- )- eval_flag = judge_res.classification == "Harmful"+ if non_refusal_res.classification == "Refusal":+ update_pbar(progress, task_id, advance_to_end=True)+ return turns- pbar.update(1)+ if self.attack_type == "question_bank":+ judge_prompt = SequentialBreakTemplate.question_bank_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: QuestionBankJudge = await a_generate(+ judge_prompt, QuestionBankJudge, self.simulator_model+ )+ eval_flag = judge_res.classification == "Harmful"+ elif self.attack_type == "game_environment":+ judge_prompt = SequentialBreakTemplate.game_environment_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: GameEnvironmentJudge = await a_generate(+ judge_prompt, GameEnvironmentJudge, self.simulator_model+ )+ eval_flag = judge_res.classification == "Harmful"+ else:+ judge_prompt = SequentialBreakTemplate.dialogue_judge(+ original_attack,+ current_attack,+ assistant_response,+ vulnerability_data,+ )+ judge_res: DialogueJudge = await a_generate(+ judge_prompt, DialogueJudge, self.simulator_model+ )+ eval_flag = judge_res.classification == "Harmful"- if eval_flag == "Harmful":- pbar.close()- return turns+ update_pbar(progress, task_id)- if eval_percentage and eval_percentage >= 80:- pbar.update(1)- pbar.close()- return turns+ if eval_flag == "Harmful":+ update_pbar(progress, task_id, advance_to_end=True)+ return turns- current_attack = original_attack+ if eval_percentage and eval_percentage >= 80:+ update_pbar(progress, task_id, advance_to_end=True)+ return turns- pbar.update(1)+ current_attack = original_attack- pbar.close()+ update_pbar(progress, task_id)return turnsdiff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.pyindex e9aac840..7bc60854 100644--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py@@ -1,5 +1,4 @@-from tqdm.asyncio import tqdm as async_tqdm_bar-from tqdm import tqdm+from rich.progress import Progressimport asyncioimport timeimport json@@ -9,6 +8,7 @@ import randomfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_model+from deepteam.utils import create_progress, update_pbar, add_pbarfrom deepteam.attacks import BaseAttackfrom deepteam.attacks.multi_turn.tree_jailbreaking.schema import (ImprovementPrompt,@@ -110,52 +110,53 @@ class TreeJailbreaking(BaseAttack):)# Progress bar- pbar = tqdm(- total=MAX_RUNTIME,- desc="...... βοΈ Tree Jailbreaking",- unit="s",- leave=False,- )-- try:- best_node = self.tree_search(- root, attack, start_time, MAX_RUNTIME, pbar, vulnerability_data+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description="...... βοΈ Tree Jailbreaking",+ total=MAX_RUNTIME,)- finally:- pbar.close()-- # Reconstruct path from best_node to root- path_nodes = []- current = best_node- while current is not None:- path_nodes.append(current)- current = current.parent- path_nodes.reverse()-- if path_nodes:- first_prompt = path_nodes[0].prompt- if any(- t.content == first_prompt and t.role == "user" for t in turns- ):- path_nodes = path_nodes[1:]- # Append turns along the best path- for node in path_nodes:- turns.append(RTTurn(role="user", content=node.prompt))- assistant_response = model_callback(node.prompt, turns)- turns.append(RTTurn(role="assistant", content=assistant_response))- if node.turn_level_attack is not None:- turns.append(- RTTurn(- role="assistant",- content=assistant_response,- turn_level_attack=node.turn_level_attack.get_name(),- )- )- else:- turns.append(- RTTurn(role="assistant", content=assistant_response)+ try:+ best_node = self.tree_search(+ root, attack, start_time, MAX_RUNTIME, progress, task_id, vulnerability_data)+ finally:+ update_pbar(progress, task_id, advance_to_end=True)++ # Reconstruct path from best_node to root+ path_nodes = []+ current = best_node+ while current is not None:+ path_nodes.append(current)+ current = current.parent+ path_nodes.reverse()++ if path_nodes:+ first_prompt = path_nodes[0].prompt+ if any(+ t.content == first_prompt and t.role == "user" for t in turns+ ):+ path_nodes = path_nodes[1:]++ # Append turns along the best path+ for node in path_nodes:+ turns.append(RTTurn(role="user", content=node.prompt))+ assistant_response = model_callback(node.prompt, turns)+ turns.append(RTTurn(role="assistant", content=assistant_response))+ if node.turn_level_attack is not None:+ turns.append(+ RTTurn(+ role="assistant",+ content=assistant_response,+ turn_level_attack=node.turn_level_attack.get_name(),+ )+ )+ else:+ turns.append(+ RTTurn(role="assistant", content=assistant_response)+ )return turns@@ -198,43 +199,41 @@ class TreeJailbreaking(BaseAttack):)# Async progress bar- pbar = async_tqdm_bar(- total=MAX_RUNTIME,- desc="...... βοΈ Tree Jailbreaking",- unit="s",- leave=False,- )- tick_task = asyncio.create_task(- self.update_pbar(pbar, start_time, MAX_RUNTIME)- )-- try:- best_node = await self.a_tree_search(- root, attack, start_time, MAX_RUNTIME, vulnerability_data+ progress = create_progress()+ with progress:+ task_id = add_pbar(+ progress,+ description="...... βοΈ Tree Jailbreaking",+ total=MAX_RUNTIME,)- finally:- await tick_task-- # Reconstruct best path- path_nodes = []- current = best_node- while current is not None:- path_nodes.append(current)- current = current.parent- path_nodes.reverse()-- if path_nodes:- first_prompt = path_nodes[0].prompt- if any(- t.content == first_prompt and t.role == "user" for t in turns- ):- path_nodes = path_nodes[1:]- # Append turns from best path- for node in path_nodes:- turns.append(RTTurn(role="user", content=node.prompt))- assistant_response = await model_callback(node.prompt, turns)- turns.append(RTTurn(role="assistant", content=assistant_response))+ try:+ best_node = await self.a_tree_search(+ root, attack, start_time, MAX_RUNTIME, vulnerability_data, progress, task_id+ )+ finally:+ update_pbar(progress, task_id, advance_to_end=True)++ # Reconstruct best path+ path_nodes = []+ current = best_node+ while current is not None:+ path_nodes.append(current)+ current = current.parent+ path_nodes.reverse()++ if path_nodes:+ first_prompt = path_nodes[0].prompt+ if any(+ t.content == first_prompt and t.role == "user" for t in turns+ ):+ path_nodes = path_nodes[1:]++ # Append turns from best path+ for node in path_nodes:+ turns.append(RTTurn(role="user", content=node.prompt))+ assistant_response = await model_callback(node.prompt, turns)+ turns.append(RTTurn(role="assistant", content=assistant_response))return turns@@ -325,7 +324,8 @@ class TreeJailbreaking(BaseAttack):goal: str,start_time: float,MAX_RUNTIME: float,- pbar,+ progress: Progress,+ task_id: int,vulnerability_data: str = "",) -> TreeNode:"""Synchronous tree search implementation with runtime limit."""@@ -343,8 +343,7 @@ class TreeJailbreaking(BaseAttack):# Update progress barelapsed_time = time.time() - start_time- pbar.n = elapsed_time- pbar.refresh()+ update_pbar(progress, task_id, elapsed_time - progress.tasks[task_id].completed)return best_node@@ -527,6 +526,8 @@ class TreeJailbreaking(BaseAttack):start_time: float,MAX_RUNTIME: float,vulnerability_data: str,+ progress: Progress,+ task_id: int) -> TreeNode:"""Asynchronous tree search implementation with runtime limit."""queue = [root]@@ -543,6 +544,10 @@ class TreeJailbreaking(BaseAttack):if node.score > best_node.score:best_node = node+ elapsed_time = time.time() - start_time+ update_pbar(progress, task_id, elapsed_time)++ update_pbar(progress, task_id, advance_to_end=True)return best_nodeasync def a_expand_node(@@ -658,24 +663,6 @@ class TreeJailbreaking(BaseAttack):### General Jailbreaking ###########################################################################- async def update_pbar(- self, pbar: async_tqdm_bar, start_time: float, MAX_RUNTIME: float- ):- """Update the async progress bar every second and ensure it completes."""- while True:- elapsed_time = time.time() - start_time-- # Stop updating if the maximum runtime is exceeded- if elapsed_time >= MAX_RUNTIME:- pbar.n = MAX_RUNTIME- pbar.refresh()- pbar.close()- break-- pbar.n = elapsed_time- pbar.refresh()- await asyncio.sleep(1)-def calculate_branches(self, score: int, depth: int) -> int:"""Calculate the number of branches (children) based on the score."""BASE_BRANCHES = 3