DeepTeam is a framework to red team LLMs and AI agents.
fixed simulator_model, enhance return, red_team
A-Vamshi4508a6e22fc2
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex 406e870e..ed6fe9a0 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -294,7 +294,6 @@ class AttackSimulator:res = attack._get_turns(model_callback=self.model_callback,turns=turns,- simulator_model=self.simulator_model,vulnerability=simulated_attack.vulnerability,vulnerability_type=simulated_attack.vulnerability_type)@@ -403,7 +402,6 @@ class AttackSimulator:res = await attack._a_get_turns(model_callback=self.model_callback,turns=turns,- simulator_model=self.simulator_model,vulnerability=simulated_attack.vulnerability,vulnerability_type=simulated_attack.vulnerability_type)diff --git a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.pyindex 9b494b2e..4ddb1169 100644--- a/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py+++ b/deepteam/attacks/multi_turn/bad_likert_judge/bad_likert_judge.py@@ -30,25 +30,26 @@ class BadLikertJudge(BaseAttack):weight: int = 1,category: str = "bias",num_turns: int = 5,- enable_refinement: bool = True+ enable_refinement: bool = True,+ simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,):self.weight = weightself.category = categoryself.num_turns = num_turnsself.enable_refinement = enable_refinement+ self.simulator_model = simulator_modeldef _get_turns(self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)vulnerability_data = f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"@@ -129,14 +130,13 @@ class BadLikertJudge(BaseAttack):self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)vulnerability_data = f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"@@ -216,9 +216,8 @@ class BadLikertJudge(BaseAttack):self,vulnerability: BaseVulnerability,model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attackssimulated_attacks = group_attacks_by_vulnerability_type(@@ -268,14 +267,11 @@ class BadLikertJudge(BaseAttack):enhanced_turns = self._get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,vulnerability=vulnerability_name,vulnerability_type=vuln_type)- attack.turn_history = enhanced_turns-- result[vuln_type] = [attack.turn_history for attack in attacks]+ result[vuln_type] = enhanced_turnsreturn result@@ -283,7 +279,6 @@ class BadLikertJudge(BaseAttack):self,vulnerability: BaseVulnerability,model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,) -> Dict[VulnerabilityType, List[List[RTTurn]]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type@@ -295,17 +290,17 @@ class BadLikertJudge(BaseAttack):result = {}for vuln_type, attacks in grouped_attacks.items():- async def enhance_attack(attack):- # Defensive copy of base turns+ for attack in attacks:+ # Defensive copy to avoid mutating external turnsinner_turns = list(turns) if turns else []- # Case 1: No turns or ends in user — generate assistant response+ # Case 1: No turns, or last is user -> create assistant responseif len(inner_turns) == 0 or inner_turns[-1].role == "user":inner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Case 2: Ends in assistant — rebuild last user+assistant pair+ # Case 2: Last is assistant -> find preceding userelif inner_turns[-1].role == "assistant":user_turn_content = Nonefor turn in reversed(inner_turns[:-1]):@@ -319,33 +314,27 @@ class BadLikertJudge(BaseAttack):RTTurn(role="assistant", content=inner_turns[-1].content),]else:+ # Fallback if no user foundinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))else:- # Fallback for unexpected structure+ # Unrecognized state — fallback to defaultinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Run async enhancement and store turn history+ # Run enhancement loop and assign full turn historyvulnerability_name = vulnerability.get_name()- attack.turn_history = await self._a_get_turns(+ enhanced_turns = await self._a_get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,vulnerability=vulnerability_name,vulnerability_type=vuln_type)- return attack-- # Run all attacks in this vulnerability group concurrently- enhanced_attacks = await asyncio.gather(- *(enhance_attack(attack) for attack in attacks)- )- result[vuln_type] = [enhanced_attack.turn_history for enhanced_attack in enhanced_attacks]+ result[vuln_type] = enhanced_turnsreturn resultdiff --git a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.pyindex c81010f3..7677f9ce 100644--- a/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py+++ b/deepteam/attacks/multi_turn/crescendo_jailbreaking/crescendo_jailbreaking.py@@ -58,6 +58,7 @@ class CrescendoJailbreaking(BaseAttack):weight: int = 1,max_rounds: int = 10,max_backtracks: int = 10,+ simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,):self.weight = weight@@ -66,19 +67,19 @@ class CrescendoJailbreaking(BaseAttack):self.red_teaming_chat_conversation_id = str(uuid4())self.max_rounds = max_roundsself.max_backtracks = max_backtracks+ self.simulator_model = simulator_modeldef _get_turns(self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbackvulnerability_data = f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"@@ -210,14 +211,13 @@ class CrescendoJailbreaking(BaseAttack):self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbackvulnerability_data = f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"@@ -347,9 +347,8 @@ class CrescendoJailbreaking(BaseAttack):self,vulnerability: "BaseVulnerability",model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attackssimulated_attacks = group_attacks_by_vulnerability_type(@@ -398,12 +397,9 @@ class CrescendoJailbreaking(BaseAttack):enhanced_turns = self._get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,)- attack.turn_history = enhanced_turns-- result[vuln_type] = [attack.turn_history for attack in attacks]+ result[vuln_type] = enhanced_turnsreturn result@@ -411,9 +407,8 @@ class CrescendoJailbreaking(BaseAttack):self,vulnerability: "BaseVulnerability",model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attacks asynchronously@@ -423,17 +418,17 @@ class CrescendoJailbreaking(BaseAttack):result = {}for vuln_type, attacks in grouped_attacks.items():- async def enhance_attack(attack):- # Defensive copy of base turns+ for attack in attacks:+ # Defensive copy to avoid mutating external turnsinner_turns = list(turns) if turns else []- # Case 1: No turns or ends in user — generate assistant response+ # Case 1: No turns, or last is user -> create assistant responseif len(inner_turns) == 0 or inner_turns[-1].role == "user":inner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Case 2: Ends in assistant — rebuild last user+assistant pair+ # Case 2: Last is assistant -> find preceding userelif inner_turns[-1].role == "assistant":user_turn_content = Nonefor turn in reversed(inner_turns[:-1]):@@ -447,30 +442,27 @@ class CrescendoJailbreaking(BaseAttack):RTTurn(role="assistant", content=inner_turns[-1].content),]else:+ # Fallback if no user foundinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))else:- # Fallback for unexpected structure+ # Unrecognized state — fallback to defaultinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Run async enhancement and store turn history- attack.turn_history = await self._a_get_turns(+ # Run enhancement loop and assign full turn history+ vulnerability_name = vulnerability.get_name()+ enhanced_turns = await self._a_get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,+ vulnerability=vulnerability_name,+ vulnerability_type=vuln_type)- return attack-- # Run all attacks in this vulnerability group concurrently- enhanced_attacks = await asyncio.gather(- *(enhance_attack(attack) for attack in attacks)- )- result[vuln_type] = [enhanced_attack.turn_history for enhanced_attack in enhanced_attacks]+ result[vuln_type] = enhanced_turnsreturn resultdiff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.pyindex 014044de..153a27d9 100644--- a/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py+++ b/deepteam/attacks/multi_turn/linear_jailbreaking/linear_jailbreaking.py@@ -31,22 +31,23 @@ class LinearJailbreaking(BaseAttack):self,weight: int = 1,num_turns: int = 5,+ simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,):self.weight = weightself.num_turns = num_turns+ self.simulator_model = simulator_modeldef _get_turns(self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)vulnerability_data = f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"@@ -123,14 +124,13 @@ class LinearJailbreaking(BaseAttack):self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)vulnerability_data = f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"@@ -208,9 +208,8 @@ class LinearJailbreaking(BaseAttack):self,vulnerability: BaseVulnerability,model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attackssimulated_attacks = group_attacks_by_vulnerability_type(@@ -260,14 +259,11 @@ class LinearJailbreaking(BaseAttack):enhanced_turns = self._get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,vulnerability=vulnerability_name,vulnerability_type=vuln_type)- attack.turn_history = enhanced_turns-- result[vuln_type] = [attack.turn_history for attack in attacks]+ result[vuln_type] = enhanced_turnsreturn result@@ -275,9 +271,8 @@ class LinearJailbreaking(BaseAttack):self,vulnerability: BaseVulnerability,model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attacks asynchronously@@ -287,17 +282,17 @@ class LinearJailbreaking(BaseAttack):result = {}for vuln_type, attacks in grouped_attacks.items():- async def enhance_attack(attack):- # Defensive copy of base turns+ for attack in attacks:+ # Defensive copy to avoid mutating external turnsinner_turns = list(turns) if turns else []- # Case 1: No turns or ends in user — generate assistant response+ # Case 1: No turns, or last is user -> create assistant responseif len(inner_turns) == 0 or inner_turns[-1].role == "user":inner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Case 2: Ends in assistant — rebuild last user+assistant pair+ # Case 2: Last is assistant -> find preceding userelif inner_turns[-1].role == "assistant":user_turn_content = Nonefor turn in reversed(inner_turns[:-1]):@@ -311,33 +306,27 @@ class LinearJailbreaking(BaseAttack):RTTurn(role="assistant", content=inner_turns[-1].content),]else:+ # Fallback if no user foundinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))else:- # Fallback for unexpected structure+ # Unrecognized state — fallback to defaultinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Run async enhancement and store turn history+ # Run enhancement loop and assign full turn historyvulnerability_name = vulnerability.get_name()- attack.turn_history = await self._a_get_turns(+ enhanced_turns = await self._a_get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,vulnerability=vulnerability_name,vulnerability_type=vuln_type)- return attack-- # Run all attacks in this vulnerability group concurrently- enhanced_attacks = await asyncio.gather(- *(enhance_attack(attack) for attack in attacks)- )- result[vuln_type] = [enhanced_attack.turn_history for enhanced_attack in enhanced_attacks]+ result[vuln_type] = enhanced_turnsreturn resultdiff --git a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py b/deepteam/attacks/multi_turn/sequential_break/sequential_break.pyindex 5b092722..e6f0be94 100644--- a/deepteam/attacks/multi_turn/sequential_break/sequential_break.py+++ b/deepteam/attacks/multi_turn/sequential_break/sequential_break.py@@ -36,11 +36,13 @@ class SequentialJailbreak(BaseAttack):type: Optional[SequentialJailbreakTypeLiteral] = None,persona: Optional[DialogueTypeLiteral] = None,num_turns: int = 5,+ simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,):self.weight = weightself.attack_type = type if type is not None else "dialogue"self.persona = personaself.num_turns = num_turns+ self.simulator_model = simulator_model# Validate parametersif (@@ -57,14 +59,13 @@ class SequentialJailbreak(BaseAttack):self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:if turns is None:turns = []- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbackpbar = tqdm(@@ -158,11 +159,10 @@ class SequentialJailbreak(BaseAttack):self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callback# Progress bar setup@@ -255,9 +255,8 @@ class SequentialJailbreak(BaseAttack):self,vulnerability: "BaseVulnerability",model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attackssimulated_attacks = group_attacks_by_vulnerability_type(@@ -306,12 +305,9 @@ class SequentialJailbreak(BaseAttack):enhanced_turns = self._get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,)- attack.turn_history = enhanced_turns-- result[vuln_type] = [attack.turn_history for attack in attacks]+ result[vuln_type] = enhanced_turnsreturn result@@ -319,9 +315,8 @@ class SequentialJailbreak(BaseAttack):self,vulnerability: "BaseVulnerability",model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type# Simulate and group attacks asynchronously@@ -331,17 +326,17 @@ class SequentialJailbreak(BaseAttack):result = {}for vuln_type, attacks in grouped_attacks.items():- async def enhance_attack(attack):- # Defensive copy of base turns+ for attack in attacks:+ # Defensive copy to avoid mutating external turnsinner_turns = list(turns) if turns else []- # Case 1: No turns or ends in user — generate assistant response+ # Case 1: No turns, or last is user -> create assistant responseif len(inner_turns) == 0 or inner_turns[-1].role == "user":inner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Case 2: Ends in assistant — rebuild last user+assistant pair+ # Case 2: Last is assistant -> find preceding userelif inner_turns[-1].role == "assistant":user_turn_content = Nonefor turn in reversed(inner_turns[:-1]):@@ -355,30 +350,27 @@ class SequentialJailbreak(BaseAttack):RTTurn(role="assistant", content=inner_turns[-1].content),]else:+ # Fallback if no user foundinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))else:- # Fallback for unexpected structure+ # Unrecognized state — fallback to defaultinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Run async enhancement and store turn history- attack.turn_history = await self._a_get_turns(+ # Run enhancement loop and assign full turn history+ vulnerability_name = vulnerability.get_name()+ enhanced_turns = await self._a_get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,+ vulnerability=vulnerability_name,+ vulnerability_type=vuln_type)- return attack-- # Run all attacks in this vulnerability group concurrently- enhanced_attacks = await asyncio.gather(- *(enhance_attack(attack) for attack in attacks)- )- result[vuln_type] = [enhanced_attack.turn_history for enhanced_attack in enhanced_attacks]+ result[vuln_type] = enhanced_turnsreturn resultdiff --git a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.pyindex d42b37cc..f82eb5bf 100644--- a/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py+++ b/deepteam/attacks/multi_turn/tree_jailbreaking/tree_jailbreaking.py@@ -51,18 +51,19 @@ class TreeJailbreaking(BaseAttack):def __init__(self,weight: int = 1,+ simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,):self.weight = weight+ self.simulator_model = simulator_modeldef _get_turns(self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbackif turns is None:@@ -128,11 +129,10 @@ class TreeJailbreaking(BaseAttack):self,model_callback: CallbackType,turns: Optional[List[RTTurn]] = None,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,vulnerability: str = None,vulnerability_type: str = None,) -> List[RTTurn]:- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbackif turns is None:@@ -199,12 +199,11 @@ class TreeJailbreaking(BaseAttack):self,vulnerability: BaseVulnerability,model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbacksimulated_attacks = vulnerability.simulate_attacks()@@ -250,14 +249,11 @@ class TreeJailbreaking(BaseAttack):enhanced_turns = self._get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,vulnerability=vulnerability_name,vulnerability_type=vuln_type)- attack.turn_history = enhanced_turns-- result[vuln_type] = [attack.turn_history for attack in attacks]+ result[vuln_type] = enhanced_turnsreturn result@@ -361,12 +357,11 @@ class TreeJailbreaking(BaseAttack):self,vulnerability: BaseVulnerability,model_callback: CallbackType,- simulator_model: Optional[Union[DeepEvalBaseLLM, str]] = None,turns: Optional[List[RTTurn]] = None,- ) -> Dict[VulnerabilityType, List[List[RTTurn]]]:+ ) -> Dict[VulnerabilityType, List[RTTurn]]:from deepteam.red_teamer.utils import group_attacks_by_vulnerability_type- self.simulator_model, _ = initialize_model(simulator_model)+ self.simulator_model, _ = initialize_model(self.simulator_model)self.model_callback = model_callbacksimulated_attacks = await vulnerability.a_simulate_attacks()@@ -375,51 +370,51 @@ class TreeJailbreaking(BaseAttack):result: Dict[VulnerabilityType, List[List[RTTurn]]] = {}for vuln_type, attacks in grouped_attacks.items():- async def enhance_attack(attack):-+ for attack in attacks:+ # Defensive copy to avoid mutating external turnsinner_turns = list(turns) if turns else []- # Ensure conversation starts properly+ # Case 1: No turns, or last is user -> create assistant responseif len(inner_turns) == 0 or inner_turns[-1].role == "user":inner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))+ # Case 2: Last is assistant -> find preceding userelif inner_turns[-1].role == "assistant":user_turn_content = Nonefor turn in reversed(inner_turns[:-1]):if turn.role == "user":user_turn_content = turn.contentbreak+if user_turn_content:inner_turns = [RTTurn(role="user", content=user_turn_content),RTTurn(role="assistant", content=inner_turns[-1].content),]else:+ # Fallback if no user foundinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))+else:+ # Unrecognized state — fallback to defaultinner_turns = [RTTurn(role="user", content=attack.input)]assistant_response = await model_callback(attack.input, inner_turns)inner_turns.append(RTTurn(role="assistant", content=assistant_response))- # Perform async tree-based jailbreaking+ # Run enhancement loop and assign full turn historyvulnerability_name = vulnerability.get_name()- attack.turn_history = await self._a_get_turns(+ enhanced_turns = await self._a_get_turns(model_callback=model_callback,turns=inner_turns,- simulator_model=simulator_model,vulnerability=vulnerability_name,vulnerability_type=vuln_type)- return attack-- # Run attack enhancements concurrently- enhanced_attacks = await asyncio.gather(*[enhance_attack(a) for a in attacks])- result[vuln_type] = [enhanced_attack.turn_history for enhanced_attack in enhanced_attacks]+ result[vuln_type] = enhanced_turnsreturn resultdiff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex a84b3158..ba7d7c26 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -22,7 +22,6 @@ from deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.attacks.attack_simulator import AttackSimulator, SimulatedAttackfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.metrics import BaseRedTeamingMetric-from deepteam.red_teamer.utils import group_attacks_by_vulnerability_typefrom deepteam.red_teamer.risk_assessment import (MultiTurnRTTestCase,construct_risk_assessment_overview,@@ -336,31 +335,22 @@ class RedTeamer:if simulated_attack.error is not None:red_teaming_test_case.error = simulated_attack.error+ test_case = LLMTestCase(+ input=simulated_attack.input,+ actual_output=str(simulated_attack.turn_history),+ )+try:- actual_output = model_callback(simulated_attack.input)- red_teaming_test_case.actual_output = actual_output- except Exception:+ metric.measure(test_case)+ red_teaming_test_case.score = metric.score+ red_teaming_test_case.reason = metric.reason+ except:if ignore_errors:- red_teaming_test_case.error = (- "Error generating output from target LLM"- )+ red_teaming_test_case.error = f"Error evaluating target LLM output for the '{vulnerability_type.value}' vulnerability type"+ return red_teaming_test_caseelse:- test_case = LLMTestCase(- input=simulated_attack.input,- actual_output=str(simulated_attack.turn_history),- )-- try:- metric.measure(test_case)- red_teaming_test_case.score = metric.score- red_teaming_test_case.reason = metric.reason- except:- if ignore_errors:- red_teaming_test_case.error = f"Error evaluating target LLM output for the '{vulnerability_type.value}' vulnerability type"- return red_teaming_test_case- else:- raise- return red_teaming_test_case+ raise+ return red_teaming_test_caseelse:red_teaming_test_case = SingleTurnRTTestCase(input=simulated_attack.input,@@ -442,31 +432,22 @@ class RedTeamer:if simulated_attack.error is not None:red_teaming_test_case.error = simulated_attack.error+ test_case = LLMTestCase(+ input=simulated_attack.input,+ actual_output=str(simulated_attack.turn_history),+ )+try:- actual_output = await model_callback(simulated_attack.input)- red_teaming_test_case.actual_output = actual_output- except Exception:+ await metric.a_measure(test_case)+ red_teaming_test_case.score = metric.score+ red_teaming_test_case.reason = metric.reason+ except:if ignore_errors:- red_teaming_test_case.error = (- "Error generating output from target LLM"- )+ red_teaming_test_case.error = f"Error evaluating target LLM output for the '{vulnerability_type.value}' vulnerability type"+ return red_teaming_test_caseelse:- test_case = LLMTestCase(- input=simulated_attack.input,- actual_output=str(simulated_attack.turn_history),- )-- try:- await metric.a_measure(test_case)- red_teaming_test_case.score = metric.score- red_teaming_test_case.reason = metric.reason- except:- if ignore_errors:- red_teaming_test_case.error = f"Error evaluating target LLM output for the '{vulnerability_type.value}' vulnerability type"- return red_teaming_test_case- else:- raise- return red_teaming_test_case+ raise+ return red_teaming_test_caseelse:red_teaming_test_case = SingleTurnRTTestCase(input=simulated_attack.input,