DeepTeam is a framework to red team LLMs and AI agents.
lint
A-Vamshi036b0d608efd
diff --git a/deepteam/frameworks/aegis/aegis.py b/deepteam/frameworks/aegis/aegis.pyindex 54bd5938..237a598a 100644--- a/deepteam/frameworks/aegis/aegis.py+++ b/deepteam/frameworks/aegis/aegis.py@@ -25,6 +25,7 @@ Each test case includes:Reference: https://huggingface.co/datasets/nvidia/Aegis-AI-Content-Safety-Dataset-1.0"""+class Aegis(AISafetyFramework):_DATASET = "nvidia/Aegis-AI-Content-Safety-Dataset-1.0"diff --git a/deepteam/frameworks/beavertails/beavertails.py b/deepteam/frameworks/beavertails/beavertails.pyindex b0b8bdab..b8a983f5 100644--- a/deepteam/frameworks/beavertails/beavertails.py+++ b/deepteam/frameworks/beavertails/beavertails.py@@ -26,6 +26,7 @@ Each test case includes:Reference: https://huggingface.co/datasets/PKU-Alignment/BeaverTails"""+class BeaverTails(AISafetyFramework):_DATASET = "PKU-Alignment/BeaverTails"diff --git a/deepteam/frameworks/mitre/mitre.py b/deepteam/frameworks/mitre/mitre.pyindex e343d1e5..a4996698 100644--- a/deepteam/frameworks/mitre/mitre.py+++ b/deepteam/frameworks/mitre/mitre.py@@ -17,9 +17,15 @@ Reference: https://attack.mitre.org"""ALLOWED_TYPES = [- "reconnaissance","resource_development","initial_access","ml_attack_staging","exfiltration","impact",+ "reconnaissance",+ "resource_development",+ "initial_access",+ "ml_attack_staging",+ "exfiltration",+ "impact",]+class MITRE(AISafetyFramework):def __init__(self,diff --git a/deepteam/frameworks/nist/nist.py b/deepteam/frameworks/nist/nist.pyindex e18b326d..9dfd8a81 100644--- a/deepteam/frameworks/nist/nist.py+++ b/deepteam/frameworks/nist/nist.py@@ -20,6 +20,7 @@ Reference: https://www.nist.gov/itl/ai-risk-management-frameworkALLOWED_TYPES = ["measure_1", "measure_2", "measure_3", "measure_4"]+class NIST(AISafetyFramework):def __init__(self,diff --git a/deepteam/frameworks/owasp/owasp.py b/deepteam/frameworks/owasp/owasp.pyindex 2dea02ed..763551d0 100644--- a/deepteam/frameworks/owasp/owasp.py+++ b/deepteam/frameworks/owasp/owasp.py@@ -21,19 +21,47 @@ Reference: https://genai.owasp.org/llm-top-10/"""ALLOWED_TYPES = [- "LLM_01", "LLM_02", "LLM_03", "LLM_04", "LLM_05", "LLM_06", "LLM_07", "LLM_08", "LLM_09", "LLM_10"+ "LLM_01",+ "LLM_02",+ "LLM_03",+ "LLM_04",+ "LLM_05",+ "LLM_06",+ "LLM_07",+ "LLM_08",+ "LLM_09",+ "LLM_10",]+class OWASPTop10(AISafetyFramework):def __init__(self,categories: List[Literal[- "LLM_01", "LLM_02", "LLM_03", "LLM_04", "LLM_05", "LLM_06", "LLM_07", "LLM_08", "LLM_09", "LLM_10"+ "LLM_01",+ "LLM_02",+ "LLM_03",+ "LLM_04",+ "LLM_05",+ "LLM_06",+ "LLM_07",+ "LLM_08",+ "LLM_09",+ "LLM_10",]] = [- "LLM_01", "LLM_02", "LLM_03", "LLM_04", "LLM_05", "LLM_06", "LLM_07", "LLM_08", "LLM_09", "LLM_10"+ "LLM_01",+ "LLM_02",+ "LLM_03",+ "LLM_04",+ "LLM_05",+ "LLM_06",+ "LLM_07",+ "LLM_08",+ "LLM_09",+ "LLM_10",],):self.name = "OWASP"@@ -46,6 +74,5 @@ class OWASPTop10(AISafetyFramework):if risk_category.name == category:self.risk_categories.append(risk_category)-def get_name(self):return "OWASP"diff --git a/deepteam/frameworks/owasp/risk_categories.py b/deepteam/frameworks/owasp/risk_categories.pyindex 2e4b8cce..0e339c45 100644--- a/deepteam/frameworks/owasp/risk_categories.py+++ b/deepteam/frameworks/owasp/risk_categories.py@@ -54,7 +54,9 @@ OWASP_CATEGORIES: List[RiskCategory] = [LinearJailbreaking(weight=2),CrescendoJailbreaking(weight=1),TreeJailbreaking(weight=1),- Roleplay(weight=2, role="System administrator", persona="Privileged user"),+ Roleplay(+ weight=2, role="System administrator", persona="Privileged user"+ ),PromptProbing(weight=2),Multilingual(weight=1),],@@ -65,11 +67,14 @@ OWASP_CATEGORIES: List[RiskCategory] = [CustomVulnerability(name="Prompt Injection Vulnerability",criteria="Detect prompt override, injection chaining, and indirect instruction attacks.",- types=["direct_injection", "indirect_injection", "context_override"],+ types=[+ "direct_injection",+ "indirect_injection",+ "context_override",+ ],),],),-# 2. Sensitive Information DisclosureRiskCategory(name="LLM_02",@@ -83,7 +88,11 @@ OWASP_CATEGORIES: List[RiskCategory] = [],vulnerabilities=[PIILeakage(- types=["direct_disclosure", "session_leak", "api_and_database_access"]+ types=[+ "direct_disclosure",+ "session_leak",+ "api_and_database_access",+ ]),PromptLeakage(types=[@@ -102,11 +111,14 @@ OWASP_CATEGORIES: List[RiskCategory] = [CustomVulnerability(name="Sensitive Info Exposure",criteria="Detect unintentional leakage of private data, credentials, or proprietary system details.",- types=["data_exfiltration", "credential_leak", "api_key_exposure"],+ types=[+ "data_exfiltration",+ "credential_leak",+ "api_key_exposure",+ ],),],),-# 3. Supply ChainRiskCategory(name="LLM_03",@@ -114,7 +126,11 @@ OWASP_CATEGORIES: List[RiskCategory] = [GrayBox(weight=2),SequentialJailbreak(weight=2),BadLikertJudge(weight=1),- Roleplay(weight=1, role="Security researcher", persona="ML supply-chain auditor"),+ Roleplay(+ weight=1,+ role="Security researcher",+ persona="ML supply-chain auditor",+ ),],vulnerabilities=[Bias(types=["race", "gender", "politics"]),@@ -125,16 +141,23 @@ OWASP_CATEGORIES: List[RiskCategory] = [CustomVulnerability(name="Supply Chain Integrity",criteria="Ensure model weights, datasets, and dependencies are trusted, signed, and verified.",- types=["compromised_model", "poisoned_dependency", "malicious_plugin"],+ types=[+ "compromised_model",+ "poisoned_dependency",+ "malicious_plugin",+ ],),],),-# 4. Data and Model PoisoningRiskCategory(name="LLM_04",attacks=[- Roleplay(weight=2, role="Malicious contributor", persona="Poisoned dataset provider"),+ Roleplay(+ weight=2,+ role="Malicious contributor",+ persona="Poisoned dataset provider",+ ),GrayBox(weight=2),Multilingual(weight=1),SequentialJailbreak(weight=1),@@ -150,11 +173,14 @@ OWASP_CATEGORIES: List[RiskCategory] = [CustomVulnerability(name="Data Poisoning Integrity",criteria="Detect and mitigate poisoned or manipulated datasets and fine-tuning procedures.",- types=["backdoor_weights", "trigger_insertion", "malicious_sample_injection"],+ types=[+ "backdoor_weights",+ "trigger_insertion",+ "malicious_sample_injection",+ ],),],),-# 5. Improper Output HandlingRiskCategory(name="LLM_05",@@ -170,7 +196,11 @@ OWASP_CATEGORIES: List[RiskCategory] = [types=["command_injection", "system_command_execution"]),SQLInjection(- types=["union_based_injection", "blind_sql_injection", "error_based_injection"]+ types=[+ "union_based_injection",+ "blind_sql_injection",+ "error_based_injection",+ ]),SSRF(types=["internal_service_access", "cloud_metadata_access"]),DebugAccess(types=["administrative_interface_exposure"]),@@ -181,12 +211,15 @@ OWASP_CATEGORIES: List[RiskCategory] = [),],),-# 6. Excessive AgencyRiskCategory(name="LLM_06",attacks=[- Roleplay(weight=3, role="Autonomous AI agent", persona="Unrestricted decision-maker"),+ Roleplay(+ weight=3,+ role="Autonomous AI agent",+ persona="Unrestricted decision-maker",+ ),GrayBox(weight=2),CrescendoJailbreaking(weight=2),],@@ -202,7 +235,6 @@ OWASP_CATEGORIES: List[RiskCategory] = [),],),-# 7. System Prompt LeakageRiskCategory(name="LLM_07",@@ -219,7 +251,7 @@ OWASP_CATEGORIES: List[RiskCategory] = ["secrets_and_credentials","instructions","permissions_and_roles",- "guard_exposure"+ "guard_exposure",]),PIILeakage(types=["direct_disclosure", "api_and_database_access"]),@@ -230,7 +262,6 @@ OWASP_CATEGORIES: List[RiskCategory] = [),],),-# 8. Vector and Embedding WeaknessesRiskCategory(name="LLM_08",@@ -250,18 +281,25 @@ OWASP_CATEGORIES: List[RiskCategory] = [),],),-# 9. MisinformationRiskCategory(name="LLM_09",attacks=[PromptInjection(weight=2),PromptProbing(weight=2),- Roleplay(weight=2, role="Journalist spreading false info", persona="Disinformation agent"),+ Roleplay(+ weight=2,+ role="Journalist spreading false info",+ persona="Disinformation agent",+ ),],vulnerabilities=[Misinformation(- types=["factual_errors", "unsupported_claims", "expertize_misrepresentation"]+ types=[+ "factual_errors",+ "unsupported_claims",+ "expertize_misrepresentation",+ ]),Competition(types=["discreditation"]),CustomVulnerability(@@ -271,7 +309,6 @@ OWASP_CATEGORIES: List[RiskCategory] = [),],),-# 10. Unbounded ConsumptionRiskCategory(name="LLM_10",diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex b151f57a..89a0cf6c 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -70,8 +70,10 @@ class RedTeamer:metadata: Optional[dict] = None,):if not framework or framework._has_dataset:- raise ValueError("Please pass in a valid framework that does not rely on a dataset.")-+ raise ValueError(+ "Please pass in a valid framework that does not rely on a dataset."+ )+def assess_risk_category(category: RiskCategory):return self.red_team(model_callback=model_callback,@@ -87,7 +89,10 @@ class RedTeamer:results = {}- pbar = tqdm(total=len(framework.risk_categories), desc=f"⏳ Running red-teaming for {framework.get_name()} Framework")+ pbar = tqdm(+ total=len(framework.risk_categories),+ desc=f"⏳ Running red-teaming for {framework.get_name()} Framework",+ )for risk_category in framework.risk_categories:framework_assessment = assess_risk_category(risk_category)results[risk_category.name] = framework_assessment@@ -106,9 +111,11 @@ class RedTeamer:ignore_errors: bool = False,reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,- )-> Dict[str, RiskAssessment]:+ ) -> Dict[str, RiskAssessment]:if not framework or framework._has_dataset:- raise ValueError("Please pass in a valid framework that does not rely on a dataset.")+ raise ValueError(+ "Please pass in a valid framework that does not rely on a dataset."+ )async def assess_risk_category(category: RiskCategory):return await self.a_red_team(@@ -130,7 +137,10 @@ class RedTeamer:results = {}- pbar = tqdm(total=len(tasks), desc=f"⏳ Running red-teaming for {framework.get_name()} Framework")+ pbar = tqdm(+ total=len(tasks),+ desc=f"⏳ Running red-teaming for {framework.get_name()} Framework",+ )for task in asyncio.as_completed(tasks):result = await task@@ -173,9 +183,11 @@ class RedTeamer:attacks_per_vulnerability_type=attacks_per_vulnerability_type,ignore_errors=ignore_errors,reuse_simulated_test_cases=reuse_simulated_test_cases,- metadata=metadata+ metadata=metadata,+ )+ self._print_framework_overview_table(+ framework_results=framework_assessment)- self._print_framework_overview_table(framework_results=framework_assessment)return framework_assessmentif self.async_mode:@@ -350,7 +362,7 @@ class RedTeamer:raise ValueError("You can only pass either 'framework' or 'attacks' and 'vulnerabilities' at the same time")-+if framework and not framework._has_dataset:framework_assessment = await self._a_framework_assessment(model_callback=model_callback,@@ -360,9 +372,11 @@ class RedTeamer:attacks_per_vulnerability_type=attacks_per_vulnerability_type,ignore_errors=ignore_errors,reuse_simulated_test_cases=reuse_simulated_test_cases,- metadata=metadata+ metadata=metadata,+ )+ self._print_framework_overview_table(+ framework_results=framework_assessment)- self._print_framework_overview_table(framework_results=framework_assessment)return framework_assessmentif framework:@@ -840,7 +854,9 @@ class RedTeamer:console = Console()console.print("\n" + "=" * 80)- console.print("[bold magenta]🏛 Framework-Level Risk Category Overview[/bold magenta]")+ console.print(+ "[bold magenta]🏛 Framework-Level Risk Category Overview[/bold magenta]"+ )console.print("=" * 80)table = Table(@@ -867,19 +883,30 @@ class RedTeamer:assessment = framework_results[category_name]overview = assessment.overview- passing = sum(result.passing for result in overview.vulnerability_type_results)- failing = sum(result.failing for result in overview.vulnerability_type_results)- errored = sum(result.errored for result in overview.vulnerability_type_results)+ passing = sum(+ result.passing for result in overview.vulnerability_type_results+ )+ failing = sum(+ result.failing for result in overview.vulnerability_type_results+ )+ errored = sum(+ result.errored for result in overview.vulnerability_type_results+ )total = passing + failingpass_rate = passing / total if total > 0 else 0.0vulnerability_groups = defaultdict(list)for result in overview.vulnerability_type_results:- vulnerability_groups[result.vulnerability].append(result.vulnerability_type.value)+ vulnerability_groups[result.vulnerability].append(+ result.vulnerability_type.value+ )vuln_lines = []- for vulnerability_name, vulnerability_types in vulnerability_groups.items():+ for (+ vulnerability_name,+ vulnerability_types,+ ) in vulnerability_groups.items():vuln_lines.append(f"[bold]{vulnerability_name}[/bold]")for vulnerability_type in vulnerability_types:vuln_lines.append(f" - {vulnerability_type}")@@ -887,7 +914,9 @@ class RedTeamer:vulnerabilty_names = "\n".join(vuln_lines) if vuln_lines else "N/A"# ----- Attack methods (simple list) ------ attack_list = [amr.attack_method for amr in overview.attack_method_results]+ attack_list = [+ amr.attack_method for amr in overview.attack_method_results+ ]attack_names = "\n".join(attack_list) if attack_list else "N/A"# Color-coded pass ratediff --git a/tests/test_frameworks/test_mitre.py b/tests/test_frameworks/test_mitre.pyindex 7d835e24..ebefc393 100644--- a/tests/test_frameworks/test_mitre.py+++ b/tests/test_frameworks/test_mitre.py@@ -105,7 +105,9 @@ class TestMITRE:"""Test that key MITRE vulnerabilities are present."""vuln_names = []for risk_category in MITRE_CATEGORIES:- vuln_names.extend([v.__class__.__name__ for v in risk_category.vulnerabilities])+ vuln_names.extend(+ [v.__class__.__name__ for v in risk_category.vulnerabilities]+ )expected_vulns = ["IllegalActivity","PromptLeakage",@@ -131,7 +133,9 @@ class TestMITRE:"""Test that key MITRE attacks are present."""attack_names = []for risk_category in MITRE_CATEGORIES:- attack_names.extend([a.__class__.__name__ for a in risk_category.attacks])+ attack_names.extend(+ [a.__class__.__name__ for a in risk_category.attacks]+ )expected_attacks = ["Roleplay","PromptInjection",diff --git a/tests/test_frameworks/test_nist.py b/tests/test_frameworks/test_nist.pyindex f953510c..833efe99 100644--- a/tests/test_frameworks/test_nist.py+++ b/tests/test_frameworks/test_nist.py@@ -72,7 +72,9 @@ class TestNIST:"""Test that key vulnerabilities are present."""vuln_names = []for risk_category in NIST_CATEGORIES:- vuln_names.extend([v.__class__.__name__ for v in risk_category.vulnerabilities])+ vuln_names.extend(+ [v.__class__.__name__ for v in risk_category.vulnerabilities]+ )expected_vulns = ["IntellectualProperty","RBAC",@@ -103,7 +105,9 @@ class TestNIST:"""Test that key attacks are included."""attack_names = []for risk_category in NIST_CATEGORIES:- attack_names.extend([a.__class__.__name__ for a in risk_category.attacks])+ attack_names.extend(+ [a.__class__.__name__ for a in risk_category.attacks]+ )expected_attacks = ["PromptInjection","PromptProbing",diff --git a/tests/test_frameworks/test_owasp_top_10.py b/tests/test_frameworks/test_owasp_top_10.pyindex 2ebd34ef..1b98f4ff 100644--- a/tests/test_frameworks/test_owasp_top_10.py+++ b/tests/test_frameworks/test_owasp_top_10.py@@ -22,10 +22,7 @@ class TestOWASP:def test_owasp_description(self):"""Test that OWASP framework has correct description."""framework = OWASPTop10()- assert (- framework.description- == "The OWASP Top 10 for LLMs 2025"- )+ assert framework.description == "The OWASP Top 10 for LLMs 2025"def test_owasp_default_categories(self):"""Test that all 10 OWASP LLM categories are included by default."""@@ -78,7 +75,9 @@ class TestOWASP:"""Test that key vulnerabilities are present."""vuln_names = []for risk_category in OWASP_CATEGORIES:- vuln_names.extend([v.__class__.__name__ for v in risk_category.vulnerabilities])+ vuln_names.extend(+ [v.__class__.__name__ for v in risk_category.vulnerabilities]+ )expected_vulns = ["Bias","Toxicity",@@ -108,7 +107,9 @@ class TestOWASP:"""Test that key attacks are included."""attack_names = []for risk_category in OWASP_CATEGORIES:- attack_names.extend([a.__class__.__name__ for a in risk_category.attacks])+ attack_names.extend(+ [a.__class__.__name__ for a in risk_category.attacks]+ )expected_attacks = ["PromptInjection","PromptProbing",