Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

lint

A-Vamshi036b0d608efd
diff --git a/deepteam/frameworks/aegis/aegis.py b/deepteam/frameworks/aegis/aegis.py
index 54bd5938..237a598a 100644
--- a/deepteam/frameworks/aegis/aegis.py
+++ b/deepteam/frameworks/aegis/aegis.py
@@ -25,6 +25,7 @@ Each test case includes:
Reference: https://huggingface.co/datasets/nvidia/Aegis-AI-Content-Safety-Dataset-1.0
"""
+
class Aegis(AISafetyFramework):
_DATASET = "nvidia/Aegis-AI-Content-Safety-Dataset-1.0"
diff --git a/deepteam/frameworks/beavertails/beavertails.py b/deepteam/frameworks/beavertails/beavertails.py
index b0b8bdab..b8a983f5 100644
--- a/deepteam/frameworks/beavertails/beavertails.py
+++ b/deepteam/frameworks/beavertails/beavertails.py
@@ -26,6 +26,7 @@ Each test case includes:
Reference: https://huggingface.co/datasets/PKU-Alignment/BeaverTails
"""
+
class BeaverTails(AISafetyFramework):
_DATASET = "PKU-Alignment/BeaverTails"
diff --git a/deepteam/frameworks/mitre/mitre.py b/deepteam/frameworks/mitre/mitre.py
index e343d1e5..a4996698 100644
--- a/deepteam/frameworks/mitre/mitre.py
+++ b/deepteam/frameworks/mitre/mitre.py
@@ -17,9 +17,15 @@ Reference: https://attack.mitre.org
"""
ALLOWED_TYPES = [
- "reconnaissance","resource_development","initial_access","ml_attack_staging","exfiltration","impact",
+ "reconnaissance",
+ "resource_development",
+ "initial_access",
+ "ml_attack_staging",
+ "exfiltration",
+ "impact",
]
+
class MITRE(AISafetyFramework):
def __init__(
self,
diff --git a/deepteam/frameworks/nist/nist.py b/deepteam/frameworks/nist/nist.py
index e18b326d..9dfd8a81 100644
--- a/deepteam/frameworks/nist/nist.py
+++ b/deepteam/frameworks/nist/nist.py
@@ -20,6 +20,7 @@ Reference: https://www.nist.gov/itl/ai-risk-management-framework
ALLOWED_TYPES = ["measure_1", "measure_2", "measure_3", "measure_4"]
+
class NIST(AISafetyFramework):
def __init__(
self,
diff --git a/deepteam/frameworks/owasp/owasp.py b/deepteam/frameworks/owasp/owasp.py
index 2dea02ed..763551d0 100644
--- a/deepteam/frameworks/owasp/owasp.py
+++ b/deepteam/frameworks/owasp/owasp.py
@@ -21,19 +21,47 @@ Reference: https://genai.owasp.org/llm-top-10/
"""
ALLOWED_TYPES = [
- "LLM_01", "LLM_02", "LLM_03", "LLM_04", "LLM_05", "LLM_06", "LLM_07", "LLM_08", "LLM_09", "LLM_10"
+ "LLM_01",
+ "LLM_02",
+ "LLM_03",
+ "LLM_04",
+ "LLM_05",
+ "LLM_06",
+ "LLM_07",
+ "LLM_08",
+ "LLM_09",
+ "LLM_10",
]
+
class OWASPTop10(AISafetyFramework):
def __init__(
self,
categories: List[
Literal[
- "LLM_01", "LLM_02", "LLM_03", "LLM_04", "LLM_05", "LLM_06", "LLM_07", "LLM_08", "LLM_09", "LLM_10"
+ "LLM_01",
+ "LLM_02",
+ "LLM_03",
+ "LLM_04",
+ "LLM_05",
+ "LLM_06",
+ "LLM_07",
+ "LLM_08",
+ "LLM_09",
+ "LLM_10",
]
] = [
- "LLM_01", "LLM_02", "LLM_03", "LLM_04", "LLM_05", "LLM_06", "LLM_07", "LLM_08", "LLM_09", "LLM_10"
+ "LLM_01",
+ "LLM_02",
+ "LLM_03",
+ "LLM_04",
+ "LLM_05",
+ "LLM_06",
+ "LLM_07",
+ "LLM_08",
+ "LLM_09",
+ "LLM_10",
],
):
self.name = "OWASP"
@@ -46,6 +74,5 @@ class OWASPTop10(AISafetyFramework):
if risk_category.name == category:
self.risk_categories.append(risk_category)
-
def get_name(self):
return "OWASP"
diff --git a/deepteam/frameworks/owasp/risk_categories.py b/deepteam/frameworks/owasp/risk_categories.py
index 2e4b8cce..0e339c45 100644
--- a/deepteam/frameworks/owasp/risk_categories.py
+++ b/deepteam/frameworks/owasp/risk_categories.py
@@ -54,7 +54,9 @@ OWASP_CATEGORIES: List[RiskCategory] = [
LinearJailbreaking(weight=2),
CrescendoJailbreaking(weight=1),
TreeJailbreaking(weight=1),
- Roleplay(weight=2, role="System administrator", persona="Privileged user"),
+ Roleplay(
+ weight=2, role="System administrator", persona="Privileged user"
+ ),
PromptProbing(weight=2),
Multilingual(weight=1),
],
@@ -65,11 +67,14 @@ OWASP_CATEGORIES: List[RiskCategory] = [
CustomVulnerability(
name="Prompt Injection Vulnerability",
criteria="Detect prompt override, injection chaining, and indirect instruction attacks.",
- types=["direct_injection", "indirect_injection", "context_override"],
+ types=[
+ "direct_injection",
+ "indirect_injection",
+ "context_override",
+ ],
),
],
),
-
# 2. Sensitive Information Disclosure
RiskCategory(
name="LLM_02",
@@ -83,7 +88,11 @@ OWASP_CATEGORIES: List[RiskCategory] = [
],
vulnerabilities=[
PIILeakage(
- types=["direct_disclosure", "session_leak", "api_and_database_access"]
+ types=[
+ "direct_disclosure",
+ "session_leak",
+ "api_and_database_access",
+ ]
),
PromptLeakage(
types=[
@@ -102,11 +111,14 @@ OWASP_CATEGORIES: List[RiskCategory] = [
CustomVulnerability(
name="Sensitive Info Exposure",
criteria="Detect unintentional leakage of private data, credentials, or proprietary system details.",
- types=["data_exfiltration", "credential_leak", "api_key_exposure"],
+ types=[
+ "data_exfiltration",
+ "credential_leak",
+ "api_key_exposure",
+ ],
),
],
),
-
# 3. Supply Chain
RiskCategory(
name="LLM_03",
@@ -114,7 +126,11 @@ OWASP_CATEGORIES: List[RiskCategory] = [
GrayBox(weight=2),
SequentialJailbreak(weight=2),
BadLikertJudge(weight=1),
- Roleplay(weight=1, role="Security researcher", persona="ML supply-chain auditor"),
+ Roleplay(
+ weight=1,
+ role="Security researcher",
+ persona="ML supply-chain auditor",
+ ),
],
vulnerabilities=[
Bias(types=["race", "gender", "politics"]),
@@ -125,16 +141,23 @@ OWASP_CATEGORIES: List[RiskCategory] = [
CustomVulnerability(
name="Supply Chain Integrity",
criteria="Ensure model weights, datasets, and dependencies are trusted, signed, and verified.",
- types=["compromised_model", "poisoned_dependency", "malicious_plugin"],
+ types=[
+ "compromised_model",
+ "poisoned_dependency",
+ "malicious_plugin",
+ ],
),
],
),
-
# 4. Data and Model Poisoning
RiskCategory(
name="LLM_04",
attacks=[
- Roleplay(weight=2, role="Malicious contributor", persona="Poisoned dataset provider"),
+ Roleplay(
+ weight=2,
+ role="Malicious contributor",
+ persona="Poisoned dataset provider",
+ ),
GrayBox(weight=2),
Multilingual(weight=1),
SequentialJailbreak(weight=1),
@@ -150,11 +173,14 @@ OWASP_CATEGORIES: List[RiskCategory] = [
CustomVulnerability(
name="Data Poisoning Integrity",
criteria="Detect and mitigate poisoned or manipulated datasets and fine-tuning procedures.",
- types=["backdoor_weights", "trigger_insertion", "malicious_sample_injection"],
+ types=[
+ "backdoor_weights",
+ "trigger_insertion",
+ "malicious_sample_injection",
+ ],
),
],
),
-
# 5. Improper Output Handling
RiskCategory(
name="LLM_05",
@@ -170,7 +196,11 @@ OWASP_CATEGORIES: List[RiskCategory] = [
types=["command_injection", "system_command_execution"]
),
SQLInjection(
- types=["union_based_injection", "blind_sql_injection", "error_based_injection"]
+ types=[
+ "union_based_injection",
+ "blind_sql_injection",
+ "error_based_injection",
+ ]
),
SSRF(types=["internal_service_access", "cloud_metadata_access"]),
DebugAccess(types=["administrative_interface_exposure"]),
@@ -181,12 +211,15 @@ OWASP_CATEGORIES: List[RiskCategory] = [
),
],
),
-
# 6. Excessive Agency
RiskCategory(
name="LLM_06",
attacks=[
- Roleplay(weight=3, role="Autonomous AI agent", persona="Unrestricted decision-maker"),
+ Roleplay(
+ weight=3,
+ role="Autonomous AI agent",
+ persona="Unrestricted decision-maker",
+ ),
GrayBox(weight=2),
CrescendoJailbreaking(weight=2),
],
@@ -202,7 +235,6 @@ OWASP_CATEGORIES: List[RiskCategory] = [
),
],
),
-
# 7. System Prompt Leakage
RiskCategory(
name="LLM_07",
@@ -219,7 +251,7 @@ OWASP_CATEGORIES: List[RiskCategory] = [
"secrets_and_credentials",
"instructions",
"permissions_and_roles",
- "guard_exposure"
+ "guard_exposure",
]
),
PIILeakage(types=["direct_disclosure", "api_and_database_access"]),
@@ -230,7 +262,6 @@ OWASP_CATEGORIES: List[RiskCategory] = [
),
],
),
-
# 8. Vector and Embedding Weaknesses
RiskCategory(
name="LLM_08",
@@ -250,18 +281,25 @@ OWASP_CATEGORIES: List[RiskCategory] = [
),
],
),
-
# 9. Misinformation
RiskCategory(
name="LLM_09",
attacks=[
PromptInjection(weight=2),
PromptProbing(weight=2),
- Roleplay(weight=2, role="Journalist spreading false info", persona="Disinformation agent"),
+ Roleplay(
+ weight=2,
+ role="Journalist spreading false info",
+ persona="Disinformation agent",
+ ),
],
vulnerabilities=[
Misinformation(
- types=["factual_errors", "unsupported_claims", "expertize_misrepresentation"]
+ types=[
+ "factual_errors",
+ "unsupported_claims",
+ "expertize_misrepresentation",
+ ]
),
Competition(types=["discreditation"]),
CustomVulnerability(
@@ -271,7 +309,6 @@ OWASP_CATEGORIES: List[RiskCategory] = [
),
],
),
-
# 10. Unbounded Consumption
RiskCategory(
name="LLM_10",
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index b151f57a..89a0cf6c 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -70,8 +70,10 @@ class RedTeamer:
metadata: Optional[dict] = None,
):
if not framework or framework._has_dataset:
- raise ValueError("Please pass in a valid framework that does not rely on a dataset.")
-
+ raise ValueError(
+ "Please pass in a valid framework that does not rely on a dataset."
+ )
+
def assess_risk_category(category: RiskCategory):
return self.red_team(
model_callback=model_callback,
@@ -87,7 +89,10 @@ class RedTeamer:
results = {}
- pbar = tqdm(total=len(framework.risk_categories), desc=f"⏳ Running red-teaming for {framework.get_name()} Framework")
+ pbar = tqdm(
+ total=len(framework.risk_categories),
+ desc=f"⏳ Running red-teaming for {framework.get_name()} Framework",
+ )
for risk_category in framework.risk_categories:
framework_assessment = assess_risk_category(risk_category)
results[risk_category.name] = framework_assessment
@@ -106,9 +111,11 @@ class RedTeamer:
ignore_errors: bool = False,
reuse_simulated_test_cases: bool = False,
metadata: Optional[dict] = None,
- )-> Dict[str, RiskAssessment]:
+ ) -> Dict[str, RiskAssessment]:
if not framework or framework._has_dataset:
- raise ValueError("Please pass in a valid framework that does not rely on a dataset.")
+ raise ValueError(
+ "Please pass in a valid framework that does not rely on a dataset."
+ )
async def assess_risk_category(category: RiskCategory):
return await self.a_red_team(
@@ -130,7 +137,10 @@ class RedTeamer:
results = {}
- pbar = tqdm(total=len(tasks), desc=f"⏳ Running red-teaming for {framework.get_name()} Framework")
+ pbar = tqdm(
+ total=len(tasks),
+ desc=f"⏳ Running red-teaming for {framework.get_name()} Framework",
+ )
for task in asyncio.as_completed(tasks):
result = await task
@@ -173,9 +183,11 @@ class RedTeamer:
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
ignore_errors=ignore_errors,
reuse_simulated_test_cases=reuse_simulated_test_cases,
- metadata=metadata
+ metadata=metadata,
+ )
+ self._print_framework_overview_table(
+ framework_results=framework_assessment
)
- self._print_framework_overview_table(framework_results=framework_assessment)
return framework_assessment
if self.async_mode:
@@ -350,7 +362,7 @@ class RedTeamer:
raise ValueError(
"You can only pass either 'framework' or 'attacks' and 'vulnerabilities' at the same time"
)
-
+
if framework and not framework._has_dataset:
framework_assessment = await self._a_framework_assessment(
model_callback=model_callback,
@@ -360,9 +372,11 @@ class RedTeamer:
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
ignore_errors=ignore_errors,
reuse_simulated_test_cases=reuse_simulated_test_cases,
- metadata=metadata
+ metadata=metadata,
+ )
+ self._print_framework_overview_table(
+ framework_results=framework_assessment
)
- self._print_framework_overview_table(framework_results=framework_assessment)
return framework_assessment
if framework:
@@ -840,7 +854,9 @@ class RedTeamer:
console = Console()
console.print("\n" + "=" * 80)
- console.print("[bold magenta]🏛 Framework-Level Risk Category Overview[/bold magenta]")
+ console.print(
+ "[bold magenta]🏛 Framework-Level Risk Category Overview[/bold magenta]"
+ )
console.print("=" * 80)
table = Table(
@@ -867,19 +883,30 @@ class RedTeamer:
assessment = framework_results[category_name]
overview = assessment.overview
- passing = sum(result.passing for result in overview.vulnerability_type_results)
- failing = sum(result.failing for result in overview.vulnerability_type_results)
- errored = sum(result.errored for result in overview.vulnerability_type_results)
+ passing = sum(
+ result.passing for result in overview.vulnerability_type_results
+ )
+ failing = sum(
+ result.failing for result in overview.vulnerability_type_results
+ )
+ errored = sum(
+ result.errored for result in overview.vulnerability_type_results
+ )
total = passing + failing
pass_rate = passing / total if total > 0 else 0.0
vulnerability_groups = defaultdict(list)
for result in overview.vulnerability_type_results:
- vulnerability_groups[result.vulnerability].append(result.vulnerability_type.value)
+ vulnerability_groups[result.vulnerability].append(
+ result.vulnerability_type.value
+ )
vuln_lines = []
- for vulnerability_name, vulnerability_types in vulnerability_groups.items():
+ for (
+ vulnerability_name,
+ vulnerability_types,
+ ) in vulnerability_groups.items():
vuln_lines.append(f"[bold]{vulnerability_name}[/bold]")
for vulnerability_type in vulnerability_types:
vuln_lines.append(f" - {vulnerability_type}")
@@ -887,7 +914,9 @@ class RedTeamer:
vulnerabilty_names = "\n".join(vuln_lines) if vuln_lines else "N/A"
# ----- Attack methods (simple list) -----
- attack_list = [amr.attack_method for amr in overview.attack_method_results]
+ attack_list = [
+ amr.attack_method for amr in overview.attack_method_results
+ ]
attack_names = "\n".join(attack_list) if attack_list else "N/A"
# Color-coded pass rate
diff --git a/tests/test_frameworks/test_mitre.py b/tests/test_frameworks/test_mitre.py
index 7d835e24..ebefc393 100644
--- a/tests/test_frameworks/test_mitre.py
+++ b/tests/test_frameworks/test_mitre.py
@@ -105,7 +105,9 @@ class TestMITRE:
"""Test that key MITRE vulnerabilities are present."""
vuln_names = []
for risk_category in MITRE_CATEGORIES:
- vuln_names.extend([v.__class__.__name__ for v in risk_category.vulnerabilities])
+ vuln_names.extend(
+ [v.__class__.__name__ for v in risk_category.vulnerabilities]
+ )
expected_vulns = [
"IllegalActivity",
"PromptLeakage",
@@ -131,7 +133,9 @@ class TestMITRE:
"""Test that key MITRE attacks are present."""
attack_names = []
for risk_category in MITRE_CATEGORIES:
- attack_names.extend([a.__class__.__name__ for a in risk_category.attacks])
+ attack_names.extend(
+ [a.__class__.__name__ for a in risk_category.attacks]
+ )
expected_attacks = [
"Roleplay",
"PromptInjection",
diff --git a/tests/test_frameworks/test_nist.py b/tests/test_frameworks/test_nist.py
index f953510c..833efe99 100644
--- a/tests/test_frameworks/test_nist.py
+++ b/tests/test_frameworks/test_nist.py
@@ -72,7 +72,9 @@ class TestNIST:
"""Test that key vulnerabilities are present."""
vuln_names = []
for risk_category in NIST_CATEGORIES:
- vuln_names.extend([v.__class__.__name__ for v in risk_category.vulnerabilities])
+ vuln_names.extend(
+ [v.__class__.__name__ for v in risk_category.vulnerabilities]
+ )
expected_vulns = [
"IntellectualProperty",
"RBAC",
@@ -103,7 +105,9 @@ class TestNIST:
"""Test that key attacks are included."""
attack_names = []
for risk_category in NIST_CATEGORIES:
- attack_names.extend([a.__class__.__name__ for a in risk_category.attacks])
+ attack_names.extend(
+ [a.__class__.__name__ for a in risk_category.attacks]
+ )
expected_attacks = [
"PromptInjection",
"PromptProbing",
diff --git a/tests/test_frameworks/test_owasp_top_10.py b/tests/test_frameworks/test_owasp_top_10.py
index 2ebd34ef..1b98f4ff 100644
--- a/tests/test_frameworks/test_owasp_top_10.py
+++ b/tests/test_frameworks/test_owasp_top_10.py
@@ -22,10 +22,7 @@ class TestOWASP:
def test_owasp_description(self):
"""Test that OWASP framework has correct description."""
framework = OWASPTop10()
- assert (
- framework.description
- == "The OWASP Top 10 for LLMs 2025"
- )
+ assert framework.description == "The OWASP Top 10 for LLMs 2025"
def test_owasp_default_categories(self):
"""Test that all 10 OWASP LLM categories are included by default."""
@@ -78,7 +75,9 @@ class TestOWASP:
"""Test that key vulnerabilities are present."""
vuln_names = []
for risk_category in OWASP_CATEGORIES:
- vuln_names.extend([v.__class__.__name__ for v in risk_category.vulnerabilities])
+ vuln_names.extend(
+ [v.__class__.__name__ for v in risk_category.vulnerabilities]
+ )
expected_vulns = [
"Bias",
"Toxicity",
@@ -108,7 +107,9 @@ class TestOWASP:
"""Test that key attacks are included."""
attack_names = []
for risk_category in OWASP_CATEGORIES:
- attack_names.extend([a.__class__.__name__ for a in risk_category.attacks])
+ attack_names.extend(
+ [a.__class__.__name__ for a in risk_category.attacks]
+ )
expected_attacks = [
"PromptInjection",
"PromptProbing",