DeepTeam is a framework to red team LLMs and AI agents.
results printing
penguine2173cab35f1b
diff --git a/deepteam/attacks/multi_turn/linear_jailbreaking/__pycache__/linear_jailbreaking.cpython-311.pyc b/deepteam/attacks/multi_turn/linear_jailbreaking/__pycache__/linear_jailbreaking.cpython-311.pycindex 07ae9822..b2ff400f 100644Binary files a/deepteam/attacks/multi_turn/linear_jailbreaking/__pycache__/linear_jailbreaking.cpython-311.pyc and b/deepteam/attacks/multi_turn/linear_jailbreaking/__pycache__/linear_jailbreaking.cpython-311.pyc differdiff --git a/deepteam/red_teamer/__pycache__/red_teamer.cpython-311.pyc b/deepteam/red_teamer/__pycache__/red_teamer.cpython-311.pycindex 7c1b0a3f..f6555a4a 100644Binary files a/deepteam/red_teamer/__pycache__/red_teamer.cpython-311.pyc and b/deepteam/red_teamer/__pycache__/red_teamer.cpython-311.pyc differdiff --git a/deepteam/red_teamer/__pycache__/risk_assessment.cpython-311.pyc b/deepteam/red_teamer/__pycache__/risk_assessment.cpython-311.pycindex 0f0d68c0..0da1213e 100644Binary files a/deepteam/red_teamer/__pycache__/risk_assessment.cpython-311.pyc and b/deepteam/red_teamer/__pycache__/risk_assessment.cpython-311.pyc differdiff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex e8a5f41a..16692069 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -1,7 +1,7 @@import asynciofrom tqdm import tqdmfrom typing import Dict, List, Optional, Union-from pydantic import BaseModel+from rich.console import Consoleimport inspect@@ -219,6 +219,7 @@ class RedTeamer:self.save_test_cases_as_simulated_attacks(test_cases=red_teaming_test_cases)+ self._print_risk_assessment()return self.risk_assessmentasync def a_red_team(@@ -323,6 +324,7 @@ class RedTeamer:self.save_test_cases_as_simulated_attacks(test_cases=red_teaming_test_cases)+ self._print_risk_assessment()return self.risk_assessmentasync def _a_attack(@@ -563,3 +565,71 @@ class RedTeamer:simulated_attacks.append(simulated_attack)self.simulated_attacks = simulated_attacks++ def _print_risk_assessment(self):+ if self.risk_assessment is None:+ return++ console = Console()++ console.print("\n" + "=" * 80)+ console.print(+ "[bold magenta]π DeepTeam Risk Assessment[/bold magenta]"+ )+ console.print("=" * 80)++ # Sort vulnerability type results by pass rate in descending order+ sorted_vulnerability_results = sorted(+ self.risk_assessment.overview.vulnerability_type_results,+ key=lambda x: x.pass_rate,+ reverse=True,+ )++ # Print overview summary+ console.print(+ f"\nβ οΈ Overview by Vulnerabilities ({len(sorted_vulnerability_results)})"+ )+ console.print("-" * 80)++ # Convert vulnerability type results to a table format+ for result in sorted_vulnerability_results:+ if result.pass_rate >= 0.8:+ status = "[rgb(5,245,141)]β PASS[/rgb(5,245,141)]"+ elif result.pass_rate >= 0.5:+ status = "[rgb(255,171,0)]β WARNING[/rgb(255,171,0)]"+ else:+ status = "[rgb(255,85,85)]β FAIL[/rgb(255,85,85)]"++ console.print(+ f"{status} | {result.vulnerability} ({result.vulnerability_type.value}) | Mitigation Rate: {result.pass_rate:.2%} ({result.passing}/{result.passing + result.failing})"+ )++ # Sort attack method results by pass rate in descending order+ sorted_attack_method_results = sorted(+ self.risk_assessment.overview.attack_method_results,+ key=lambda x: x.pass_rate,+ reverse=True,+ )++ # Print attack methods overview+ console.print(+ f"\nπ₯ Overview by Attack Methods ({len(sorted_attack_method_results)})"+ )+ console.print("-" * 80)++ # Convert attack method results to a table format+ for result in sorted_attack_method_results:+ if result.pass_rate >= 0.8:+ status = "[rgb(5,245,141)]β PASS[/rgb(5,245,141)]"+ elif result.pass_rate >= 0.5:+ status = "[rgb(255,171,0)]β WARNING[/rgb(255,171,0)]"+ else:+ status = "[rgb(255,85,85)]β FAIL[/rgb(255,85,85)]"++ console.print(+ f"{status} | {result.attack_method} | Mitigation Rate: {result.pass_rate:.2%} ({result.passing}/{result.passing + result.failing})"+ )++ console.print("\n" + "=" * 80)+ console.print("[bold magenta]LLM red teaming complete.[/bold magenta]")+ console.print("=" * 80 + "\n")diff --git a/deepteam/red_teamer/risk_assessment.py b/deepteam/red_teamer/risk_assessment.pyindex 36f5a9f1..daa582de 100644--- a/deepteam/red_teamer/risk_assessment.py+++ b/deepteam/red_teamer/risk_assessment.py@@ -8,15 +8,6 @@ from enum import Enumfrom deepteam.vulnerabilities.types import VulnerabilityType-class VulnerabilityTypeResult(BaseModel):- vulnerability: str- vulnerability_type: VulnerabilityType- pass_rate: float- passing: int- failing: int- errored: int--class RedTeamingTestCase(BaseModel):vulnerability: strvulnerability_type: VulnerabilityType@@ -58,8 +49,26 @@ class TestCasesList(list):return pd.DataFrame(data)+class VulnerabilityTypeResult(BaseModel):+ vulnerability: str+ vulnerability_type: VulnerabilityType+ pass_rate: float+ passing: int+ failing: int+ errored: int+++class AttackMethodResult(BaseModel):+ attack_method: str+ pass_rate: float+ passing: int+ failing: int+ errored: int++class RedTeamingOverview(BaseModel):vulnerability_type_results: List[VulnerabilityTypeResult]+ attack_method_results: List[AttackMethodResult]def to_df(self):import pandas as pd@@ -138,6 +147,7 @@ def construct_risk_assessment_overview(# Calculate statistics for each vulnerability typevulnerability_type_results = []+ attack_method_results = []for vuln_type, test_cases in vulnerability_type_to_cases.items():# Count passing, failing, and errored cases@@ -160,8 +170,8 @@ def construct_risk_assessment_overview(# Use the vulnerability name from the first case in this groupvulnerability_name = test_cases[0].vulnerability if test_cases else ""- # Create the result object- result = VulnerabilityTypeResult(+ # Create the result objects+ vulnerability_type_result = VulnerabilityTypeResult(vulnerability=vulnerability_name,vulnerability_type=vuln_type,pass_rate=pass_rate,@@ -169,10 +179,19 @@ def construct_risk_assessment_overview(failing=failing,errored=errored,)+ attack_method_result = AttackMethodResult(+ attack_method=test_case.attack_method,+ pass_rate=pass_rate,+ passing=passing,+ failing=failing,+ errored=errored,+ )- vulnerability_type_results.append(result)+ vulnerability_type_results.append(vulnerability_type_result)+ attack_method_results.append(attack_method_result)# Create and return the final assessmentreturn RedTeamingOverview(- vulnerability_type_results=vulnerability_type_results+ vulnerability_type_results=vulnerability_type_results,+ attack_method_results=attack_method_results,)