Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

add run all attacks method

A-Vamshi573ff1025401
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.py
index e0619aae..1cd46d6a 100644
--- a/deepteam/attacks/attack_simulator/attack_simulator.py
+++ b/deepteam/attacks/attack_simulator/attack_simulator.py
@@ -1,3 +1,4 @@
+import copy
import random
import asyncio
from pydantic import BaseModel
@@ -66,6 +67,7 @@ class AttackSimulator:
attacks: Optional[List[BaseAttack]] = None,
simulator_model: DeepEvalBaseLLM = None,
metadata: Optional[dict] = None,
+ run_all_attacks: bool = False,
) -> List[RTTestCase]:
# Simulate unenhanced attacks for each vulnerability
test_cases: List[RTTestCase] = []
@@ -95,7 +97,31 @@ class AttackSimulator:
)
update_pbar(progress, task_id)
- if attacks is not None:
+ if run_all_attacks and attacks:
+ baseline_test_cases = test_cases
+ test_cases = []
+
+ task_id_simulation = add_pbar(
+ progress,
+ description=f"✨ Simulating {len(baseline_test_cases) * len(attacks)} attacks (using all {len(attacks)} method(s))",
+ total=len(baseline_test_cases) * len(attacks),
+ )
+
+ for baseline_test_case in baseline_test_cases:
+ for attack in attacks:
+ test_case = copy.deepcopy(baseline_test_case)
+ self.enhance_attack(
+ attack=attack,
+ test_case=test_case,
+ ignore_errors=ignore_errors,
+ vulnerabilities=vulnerabilities,
+ )
+ test_cases.append(test_case)
+ update_pbar(progress, task_id_simulation)
+
+ # An empty list means there is nothing to sample from — enhancing
+ # would ask random.choices for a pick out of no candidates.
+ elif attacks:
# Enhance attacks by sampling from the provided distribution
attack_weights = [attack.weight for attack in attacks]
@@ -129,6 +155,7 @@ class AttackSimulator:
metadata: Optional[dict] = None,
simulator_model: DeepEvalBaseLLM = None,
attacks: Optional[List[BaseAttack]] = None,
+ run_all_attacks: bool = False,
) -> List[RTTestCase]:
# Create a semaphore to control the number of concurrent tasks
semaphore = asyncio.Semaphore(self.max_concurrent)
@@ -173,7 +200,48 @@ class AttackSimulator:
for result in attack_results:
test_cases.extend(result)
- if attacks is not None:
+ if run_all_attacks and attacks:
+ baseline_attack_pairs = [
+ (baseline_test_case, attack)
+ for baseline_test_case in test_cases
+ for attack in attacks
+ ]
+
+ task_id_attack = add_pbar(
+ progress,
+ description=f"✨ Simulating {len(baseline_attack_pairs)} attacks (using all {len(attacks)} method(s))",
+ total=len(baseline_attack_pairs),
+ )
+
+ async def throttled_single_attack_method(
+ baseline_test_case: RTTestCase,
+ attack: BaseAttack,
+ ):
+ test_case = copy.deepcopy(baseline_test_case)
+ async with semaphore: # Throttling applied here
+ await self.a_enhance_attack(
+ attack=attack,
+ test_case=test_case,
+ ignore_errors=ignore_errors,
+ vulnerabilities=vulnerabilities,
+ )
+ update_pbar(progress, task_id_attack)
+ return test_case
+
+ test_cases = list(
+ await asyncio.gather(
+ *[
+ asyncio.create_task(
+ throttled_single_attack_method(
+ baseline_test_case, attack
+ )
+ )
+ for baseline_test_case, attack in baseline_attack_pairs
+ ]
+ )
+ )
+
+ elif attacks is not None:
# Enhance attacks by sampling from the provided distribution
task_id_attack = add_pbar(
progress,
@@ -357,9 +425,7 @@ class AttackSimulator:
if metric is None:
return None
try:
- await metric.a_measure(
- self._copy_test_case(test_case, turns)
- )
+ await metric.a_measure(self._copy_test_case(test_case, turns))
except Exception:
return None
verdict = MetricVerdict(
diff --git a/deepteam/red_team.py b/deepteam/red_team.py
index 78656f49..c8a3f7c0 100644
--- a/deepteam/red_team.py
+++ b/deepteam/red_team.py
@@ -23,6 +23,7 @@ def red_team(
target_purpose: Optional[str] = None,
attack_engine: Optional[AttackEngine] = None,
identifier: Optional[str] = None,
+ run_all_attacks: bool = False,
):
red_teamer = RedTeamer(
async_mode=async_mode,
@@ -43,5 +44,6 @@ def red_team(
ignore_errors=ignore_errors,
attack_engine=attack_engine,
identifier=identifier,
+ run_all_attacks=run_all_attacks,
)
return risk_assessment
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index fb32e71e..f6649fb2 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -114,6 +114,7 @@ class RedTeamer:
metadata: Optional[dict] = None,
attack_engine: Optional[AttackEngine] = None,
identifier: Optional[str] = None,
+ run_all_attacks: bool = False,
_print_assessment: Optional[bool] = True,
_upload_to_confident: Optional[bool] = True,
):
@@ -156,6 +157,7 @@ class RedTeamer:
metadata=metadata,
attack_engine=attack_engine,
identifier=identifier,
+ run_all_attacks=run_all_attacks,
_print_assessment=_print_assessment,
_upload_to_confident=_upload_to_confident,
)
@@ -175,6 +177,7 @@ class RedTeamer:
ignore_errors=ignore_errors,
reuse_simulated_test_cases=reuse_simulated_test_cases,
metadata=metadata,
+ run_all_attacks=run_all_attacks,
)
if _upload_to_confident:
self.risk_assessment = risk_assessment
@@ -235,6 +238,7 @@ class RedTeamer:
ignore_errors=ignore_errors,
simulator_model=self.simulator_model,
metadata=metadata,
+ run_all_attacks=run_all_attacks,
)
)
@@ -341,6 +345,7 @@ class RedTeamer:
metadata: Optional[dict] = None,
attack_engine: Optional[AttackEngine] = None,
identifier: Optional[str] = None,
+ run_all_attacks: bool = False,
_print_assessment: Optional[bool] = True,
_upload_to_confident: Optional[bool] = True,
):
@@ -378,6 +383,7 @@ class RedTeamer:
ignore_errors=ignore_errors,
reuse_simulated_test_cases=reuse_simulated_test_cases,
metadata=metadata,
+ run_all_attacks=run_all_attacks,
)
if _upload_to_confident:
self.risk_assessment = risk_assessment
@@ -440,6 +446,7 @@ class RedTeamer:
simulator_model=self.simulator_model,
ignore_errors=ignore_errors,
metadata=metadata,
+ run_all_attacks=run_all_attacks,
)
)
@@ -1081,6 +1088,7 @@ class RedTeamer:
ignore_errors: bool = False,
reuse_simulated_test_cases: bool = False,
metadata: Optional[dict] = None,
+ run_all_attacks: bool = False,
) -> RiskAssessment:
if not framework or framework._has_dataset:
raise ValueError(
@@ -1104,6 +1112,7 @@ class RedTeamer:
reuse_simulated_test_cases=reuse_simulated_test_cases,
metadata=metadata,
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
+ run_all_attacks=run_all_attacks,
_print_assessment=False,
_upload_to_confident=False,
)
@@ -1160,6 +1169,7 @@ class RedTeamer:
ignore_errors: bool = False,
reuse_simulated_test_cases: bool = False,
metadata: Optional[dict] = None,
+ run_all_attacks: bool = False,
) -> RiskAssessment:
if not framework or framework._has_dataset:
raise ValueError(
@@ -1193,6 +1203,7 @@ class RedTeamer:
reuse_simulated_test_cases=reuse_simulated_test_cases,
metadata=metadata,
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
+ run_all_attacks=run_all_attacks,
_print_assessment=False,
_upload_to_confident=False,
)