DeepTeam is a framework to red team LLMs and AI agents.
add run all attacks method
A-Vamshi573ff1025401
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex e0619aae..1cd46d6a 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -1,3 +1,4 @@+import copyimport randomimport asynciofrom pydantic import BaseModel@@ -66,6 +67,7 @@ class AttackSimulator:attacks: Optional[List[BaseAttack]] = None,simulator_model: DeepEvalBaseLLM = None,metadata: Optional[dict] = None,+ run_all_attacks: bool = False,) -> List[RTTestCase]:# Simulate unenhanced attacks for each vulnerabilitytest_cases: List[RTTestCase] = []@@ -95,7 +97,31 @@ class AttackSimulator:)update_pbar(progress, task_id)- if attacks is not None:+ if run_all_attacks and attacks:+ baseline_test_cases = test_cases+ test_cases = []++ task_id_simulation = add_pbar(+ progress,+ description=f"✨ Simulating {len(baseline_test_cases) * len(attacks)} attacks (using all {len(attacks)} method(s))",+ total=len(baseline_test_cases) * len(attacks),+ )++ for baseline_test_case in baseline_test_cases:+ for attack in attacks:+ test_case = copy.deepcopy(baseline_test_case)+ self.enhance_attack(+ attack=attack,+ test_case=test_case,+ ignore_errors=ignore_errors,+ vulnerabilities=vulnerabilities,+ )+ test_cases.append(test_case)+ update_pbar(progress, task_id_simulation)++ # An empty list means there is nothing to sample from — enhancing+ # would ask random.choices for a pick out of no candidates.+ elif attacks:# Enhance attacks by sampling from the provided distributionattack_weights = [attack.weight for attack in attacks]@@ -129,6 +155,7 @@ class AttackSimulator:metadata: Optional[dict] = None,simulator_model: DeepEvalBaseLLM = None,attacks: Optional[List[BaseAttack]] = None,+ run_all_attacks: bool = False,) -> List[RTTestCase]:# Create a semaphore to control the number of concurrent taskssemaphore = asyncio.Semaphore(self.max_concurrent)@@ -173,7 +200,48 @@ class AttackSimulator:for result in attack_results:test_cases.extend(result)- if attacks is not None:+ if run_all_attacks and attacks:+ baseline_attack_pairs = [+ (baseline_test_case, attack)+ for baseline_test_case in test_cases+ for attack in attacks+ ]++ task_id_attack = add_pbar(+ progress,+ description=f"✨ Simulating {len(baseline_attack_pairs)} attacks (using all {len(attacks)} method(s))",+ total=len(baseline_attack_pairs),+ )++ async def throttled_single_attack_method(+ baseline_test_case: RTTestCase,+ attack: BaseAttack,+ ):+ test_case = copy.deepcopy(baseline_test_case)+ async with semaphore: # Throttling applied here+ await self.a_enhance_attack(+ attack=attack,+ test_case=test_case,+ ignore_errors=ignore_errors,+ vulnerabilities=vulnerabilities,+ )+ update_pbar(progress, task_id_attack)+ return test_case++ test_cases = list(+ await asyncio.gather(+ *[+ asyncio.create_task(+ throttled_single_attack_method(+ baseline_test_case, attack+ )+ )+ for baseline_test_case, attack in baseline_attack_pairs+ ]+ )+ )++ elif attacks is not None:# Enhance attacks by sampling from the provided distributiontask_id_attack = add_pbar(progress,@@ -357,9 +425,7 @@ class AttackSimulator:if metric is None:return Nonetry:- await metric.a_measure(- self._copy_test_case(test_case, turns)- )+ await metric.a_measure(self._copy_test_case(test_case, turns))except Exception:return Noneverdict = MetricVerdict(diff --git a/deepteam/red_team.py b/deepteam/red_team.pyindex 78656f49..c8a3f7c0 100644--- a/deepteam/red_team.py+++ b/deepteam/red_team.py@@ -23,6 +23,7 @@ def red_team(target_purpose: Optional[str] = None,attack_engine: Optional[AttackEngine] = None,identifier: Optional[str] = None,+ run_all_attacks: bool = False,):red_teamer = RedTeamer(async_mode=async_mode,@@ -43,5 +44,6 @@ def red_team(ignore_errors=ignore_errors,attack_engine=attack_engine,identifier=identifier,+ run_all_attacks=run_all_attacks,)return risk_assessmentdiff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex fb32e71e..f6649fb2 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -114,6 +114,7 @@ class RedTeamer:metadata: Optional[dict] = None,attack_engine: Optional[AttackEngine] = None,identifier: Optional[str] = None,+ run_all_attacks: bool = False,_print_assessment: Optional[bool] = True,_upload_to_confident: Optional[bool] = True,):@@ -156,6 +157,7 @@ class RedTeamer:metadata=metadata,attack_engine=attack_engine,identifier=identifier,+ run_all_attacks=run_all_attacks,_print_assessment=_print_assessment,_upload_to_confident=_upload_to_confident,)@@ -175,6 +177,7 @@ class RedTeamer:ignore_errors=ignore_errors,reuse_simulated_test_cases=reuse_simulated_test_cases,metadata=metadata,+ run_all_attacks=run_all_attacks,)if _upload_to_confident:self.risk_assessment = risk_assessment@@ -235,6 +238,7 @@ class RedTeamer:ignore_errors=ignore_errors,simulator_model=self.simulator_model,metadata=metadata,+ run_all_attacks=run_all_attacks,))@@ -341,6 +345,7 @@ class RedTeamer:metadata: Optional[dict] = None,attack_engine: Optional[AttackEngine] = None,identifier: Optional[str] = None,+ run_all_attacks: bool = False,_print_assessment: Optional[bool] = True,_upload_to_confident: Optional[bool] = True,):@@ -378,6 +383,7 @@ class RedTeamer:ignore_errors=ignore_errors,reuse_simulated_test_cases=reuse_simulated_test_cases,metadata=metadata,+ run_all_attacks=run_all_attacks,)if _upload_to_confident:self.risk_assessment = risk_assessment@@ -440,6 +446,7 @@ class RedTeamer:simulator_model=self.simulator_model,ignore_errors=ignore_errors,metadata=metadata,+ run_all_attacks=run_all_attacks,))@@ -1081,6 +1088,7 @@ class RedTeamer:ignore_errors: bool = False,reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,+ run_all_attacks: bool = False,) -> RiskAssessment:if not framework or framework._has_dataset:raise ValueError(@@ -1104,6 +1112,7 @@ class RedTeamer:reuse_simulated_test_cases=reuse_simulated_test_cases,metadata=metadata,attacks_per_vulnerability_type=attacks_per_vulnerability_type,+ run_all_attacks=run_all_attacks,_print_assessment=False,_upload_to_confident=False,)@@ -1160,6 +1169,7 @@ class RedTeamer:ignore_errors: bool = False,reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,+ run_all_attacks: bool = False,) -> RiskAssessment:if not framework or framework._has_dataset:raise ValueError(@@ -1193,6 +1203,7 @@ class RedTeamer:reuse_simulated_test_cases=reuse_simulated_test_cases,metadata=metadata,attacks_per_vulnerability_type=attacks_per_vulnerability_type,+ run_all_attacks=run_all_attacks,_print_assessment=False,_upload_to_confident=False,)