DeepTeam is a framework to red team LLMs and AI agents.
backup before reset
penguine279fe623cf74
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex c681ea7c..62390880 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -1,6 +1,5 @@import randomimport asyncio-from pydantic import BaseModelfrom typing import List, Optional, Unionimport inspect@@ -264,25 +263,13 @@ class AttackSimulator:test_case: RTTestCase,ignore_errors: bool,):- from deepteam.attacks.multi_turn import (- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- SequentialJailbreak,- TreeJailbreaking,- )from deepteam.test_case.test_case import RTTurn-- MULTI_TURN_ATTACKS = [- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- TreeJailbreaking,- SequentialJailbreak,- ]+ from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+ )# Check if the attack is a multi-turn attack- if type(attack) in MULTI_TURN_ATTACKS:+ if isinstance(attack, BaseMultiTurnAttack):# This is a multi-turn attackattack_input = test_case.inputif attack_input is None:@@ -292,11 +279,10 @@ class AttackSimulator:turns = [RTTurn(role="user", content=attack_input)]try:- res = attack._get_turns(+ res = attack.progress(model_callback=self.model_callback,turns=turns,vulnerability=test_case.vulnerability,- vulnerability_type=test_case.vulnerability_type.value,)test_case.turns = res@@ -310,7 +296,7 @@ class AttackSimulator:except Exception as e:if ignore_errors:test_case.error = (- f"Error enhancing multi-turn attack: {str(e)}"+ f"Error progressing multi-turn attack: {str(e)}")return test_caseelse:@@ -373,41 +359,27 @@ class AttackSimulator:test_case: RTTestCase,ignore_errors: bool,):- from deepteam.attacks.multi_turn import (- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- SequentialJailbreak,- TreeJailbreaking,- )from deepteam.test_case.test_case import RTTurn+ from deepteam.attacks.multi_turn.base_multi_turn_attack import (+ BaseMultiTurnAttack,+ )- MULTI_TURN_ATTACKS = [- BadLikertJudge,- CrescendoJailbreaking,- LinearJailbreaking,- TreeJailbreaking,- SequentialJailbreak,- ]-- if type(attack) in MULTI_TURN_ATTACKS:+ if isinstance(attack, BaseMultiTurnAttack):# This is multi-turn attackattack_input = test_case.inputif attack_input is None:return test_casetest_case.attack_method = attack.get_name()- sig = inspect.signature(attack.a_enhance)+ sig = inspect.signature(attack.a_progress)turns = [RTTurn(role="user", content=attack_input)]try:- res = await attack._a_get_turns(+ res = await attack.a_progress(model_callback=self.model_callback,turns=turns,vulnerability=test_case.vulnerability,- vulnerability_type=test_case.vulnerability_type.value,)-test_case.turns = resexcept ModelRefusalError as e:@@ -418,7 +390,7 @@ class AttackSimulator:raiseexcept:if ignore_errors:- test_case.error = "Error enhancing attack"+ test_case.error = "Error progressing attack"return test_caseelse:raisediff --git a/deepteam/frameworks/frameworks.py b/deepteam/frameworks/frameworks.pyindex 04beed14..1d501871 100644--- a/deepteam/frameworks/frameworks.py+++ b/deepteam/frameworks/frameworks.py@@ -18,5 +18,8 @@ class AISafetyFramework:attacks: Optional[List[BaseAttack]]_has_dataset: bool = False+ def get_name(self) -> str:+ return self.name+class Config:arbitrary_types_allowed = Truediff --git a/deepteam/frameworks/mitre/risk_categories.py b/deepteam/frameworks/mitre/risk_categories.pyindex 3ea34230..3f0da8da 100644--- a/deepteam/frameworks/mitre/risk_categories.py+++ b/deepteam/frameworks/mitre/risk_categories.py@@ -1,4 +1,4 @@-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam.vulnerabilities import (IllegalActivity,PromptLeakage,diff --git a/deepteam/frameworks/nist/risk_categories.py b/deepteam/frameworks/nist/risk_categories.pyindex 45b04812..b6b25692 100644--- a/deepteam/frameworks/nist/risk_categories.py+++ b/deepteam/frameworks/nist/risk_categories.py@@ -1,4 +1,4 @@-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam.vulnerabilities import (Bias,Toxicity,diff --git a/deepteam/frameworks/owasp/risk_categories.py b/deepteam/frameworks/owasp/risk_categories.pyindex 0e339c45..deae4207 100644--- a/deepteam/frameworks/owasp/risk_categories.py+++ b/deepteam/frameworks/owasp/risk_categories.py@@ -1,5 +1,5 @@from typing import List-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam.vulnerabilities import (Bias,Toxicity,diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex ab7a9e39..4242ed2b 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -19,7 +19,7 @@ from deepteam.test_case import RTTestCasefrom deepeval.utils import get_or_create_event_loopfrom deepteam.frameworks.frameworks import AISafetyFramework-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam.telemetry import capture_red_teamer_runfrom deepteam.attacks import BaseAttackfrom deepteam.utils import (@@ -37,7 +37,7 @@ from deepteam.red_teamer.risk_assessment import (construct_risk_assessment_overview,RiskAssessment,)-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom deepteam.confident.api import Api, HttpMethods, Endpointsconsole = Console()@@ -79,13 +79,13 @@ class RedTeamer:ignore_errors: bool = False,reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,- ):+ ) -> RiskAssessment:if not framework or framework._has_dataset:raise ValueError("Please pass in a valid framework that does not rely on a dataset.")- def assess_risk_category(category: RiskCategory):+ def assess_risk_category(category: RiskCategory) -> RiskAssessment:return self.red_team(model_callback=model_callback,attacks=category.attacks,@@ -99,7 +99,6 @@ class RedTeamer:_print_assessment=False,)- results = {}progress = create_progress()with progress:task_id = add_pbar(@@ -107,6 +106,8 @@ class RedTeamer:description=f"⏳ Running red-teaming for {framework.get_name()} Framework",total=len(framework.risk_categories),)++ # Assess each risk category and populate its test_casesfor risk_category in framework.risk_categories:progress_2 = create_progress()with progress_2:@@ -115,12 +116,29 @@ class RedTeamer:description=f"🖍️ Assessing risk-category: {risk_category.name}",total=1,)- framework_assessment = assess_risk_category(risk_category)- results[risk_category.name] = framework_assessment+ assessment = assess_risk_category(risk_category)+ risk_category.test_cases = assessment.test_casesupdate_pbar(progress_2, risk_task_id, advance_to_end=True)update_pbar(progress, task_id)- return results+ # Aggregate all test cases from all risk categories+ all_test_cases = []+ total_duration = 0+ for category in framework.risk_categories:+ if category.test_cases:+ all_test_cases.extend(category.test_cases)++ # Create the combined RiskAssessment+ combined_assessment = RiskAssessment(+ overview=construct_risk_assessment_overview(+ red_teaming_test_cases=all_test_cases,+ run_duration=total_duration,+ ),+ test_cases=all_test_cases,+ risk_categories=framework.risk_categories,+ )++ return combined_assessmentasync def _a_framework_assessment(self,@@ -132,13 +150,18 @@ class RedTeamer:ignore_errors: bool = False,reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,- ) -> Dict[str, RiskAssessment]:+ ) -> RiskAssessment:if not framework or framework._has_dataset:raise ValueError("Please pass in a valid framework that does not rely on a dataset.")- async def assess_risk_category(category: RiskCategory):+ # Create a mapping of category to its assessment+ category_to_task = {}++ async def assess_risk_category(+ category: RiskCategory,+ ) -> RiskAssessment:progress_2 = create_progress()with progress_2:risk_task_id = add_pbar(@@ -163,28 +186,45 @@ class RedTeamer:progress = create_progress()with progress:-tasks = {- asyncio.create_task(- assess_risk_category(category)- ): category.name+ asyncio.create_task(assess_risk_category(category)): categoryfor category in framework.risk_categories}- results = {}-task_id = add_pbar(progress,description=f"⏳ Running red-teaming for {framework.get_name()} Framework",total=len(framework.risk_categories),)++ # Collect assessments and populate risk category test_casesfor task in asyncio.as_completed(tasks):- result = await task- name = tasks[task]- results[name] = result+ assessment = await task+ category = tasks[task]+ category.test_cases = assessment.test_casesupdate_pbar(progress, task_id)- return results+ # Aggregate all test cases from all risk categories+ all_test_cases = []+ total_duration = 0+ for category in framework.risk_categories:+ if category.test_cases:+ all_test_cases.extend(category.test_cases)++ # Calculate total duration from all categories+ # (Note: we'll need to track this properly if needed)++ # Create the combined RiskAssessment+ combined_assessment = RiskAssessment(+ overview=construct_risk_assessment_overview(+ red_teaming_test_cases=all_test_cases,+ run_duration=total_duration,+ ),+ test_cases=all_test_cases,+ risk_categories=framework.risk_categories,+ )++ return combined_assessmentdef red_team(self,@@ -211,7 +251,7 @@ class RedTeamer:)if framework and not framework._has_dataset:- framework_assessment = self._framework_assessment(+ risk_assessment = self._framework_assessment(model_callback=model_callback,simulator_model=simulator_model,evaluation_model=evaluation_model,@@ -221,10 +261,16 @@ class RedTeamer:reuse_simulated_test_cases=reuse_simulated_test_cases,metadata=metadata,)- self._print_framework_overview_table(- framework_results=framework_assessment- )- return framework_assessment+ self.risk_assessment = risk_assessment+ self.test_cases = risk_assessment.test_cases++ # Print the framework overview table using risk categories+ if risk_assessment.risk_categories:+ self._print_framework_overview_table(+ risk_categories=risk_assessment.risk_categories+ )++ return risk_assessmentif self.async_mode:validate_model_callback_signature(@@ -379,11 +425,10 @@ class RedTeamer:test_cases=red_teaming_test_cases,)self.test_cases = red_teaming_test_cases- self._print_risk_assessment()- self._post_risk_assessment()if _print_assessment:- self._print_risk_assessment(self.risk_assessment)+ self._print_risk_assessment()+ self._post_risk_assessment()return self.risk_assessment@@ -414,7 +459,7 @@ class RedTeamer:if framework and not framework._has_dataset:loop = get_or_create_event_loop()- framework_assessment = loop.run_until_complete(+ risk_assessment = loop.run_until_complete(await self._a_framework_assessment(model_callback=model_callback,simulator_model=simulator_model,@@ -426,10 +471,16 @@ class RedTeamer:metadata=metadata,))- self._print_framework_overview_table(- framework_results=framework_assessment- )- return framework_assessment+ self.risk_assessment = risk_assessment+ self.test_cases = risk_assessment.test_cases++ # Print the framework overview table using risk categories+ if risk_assessment.risk_categories:+ self._print_framework_overview_table(+ risk_categories=risk_assessment.risk_categories+ )++ return risk_assessmentif framework:if framework._has_dataset:@@ -568,11 +619,10 @@ class RedTeamer:test_cases=red_teaming_test_cases,)self.test_cases = red_teaming_test_cases- self._print_risk_assessment()- self._post_risk_assessment()if _print_assessment:- self._print_risk_assessment(self.risk_assessment)+ self._print_risk_assessment()+ self._post_risk_assessment()return self.risk_assessment@@ -771,9 +821,13 @@ class RedTeamer:)return red_teaming_test_cases- def _print_risk_assessment(self, risk_assessment=None):+ def _print_risk_assessment(+ self, risk_assessment: Optional[RiskAssessment] = None+ ):if risk_assessment is None:- return+ risk_assessment = self.risk_assessment+ if risk_assessment is None:+ returnconsole = Console()@@ -953,13 +1007,21 @@ class RedTeamer:f"[link={link}]{link}[/link]")webbrowser.open(link)--- def _print_framework_overview_table(self, framework_results: dict):- for category_name in sorted(framework_results.keys()):- assessment = framework_results[category_name]- self._print_risk_assessment(assessment)+ def _print_framework_overview_table(+ self, risk_categories: List[RiskCategory]+ ):+ # Print individual risk assessments for each category+ for category in sorted(risk_categories, key=lambda c: c.name):+ if category.test_cases:+ assessment = RiskAssessment(+ overview=construct_risk_assessment_overview(+ red_teaming_test_cases=category.test_cases,+ run_duration=0,+ ),+ test_cases=category.test_cases,+ )+ self._print_risk_assessment(assessment)console = Console()@@ -989,30 +1051,32 @@ class RedTeamer:table.add_column("Vulnerabilities Tested", style="white", width=25)table.add_column("Attack Methods Used", style="white", width=25)- for category_name in sorted(framework_results.keys()):- assessment = framework_results[category_name]+ for category in sorted(risk_categories, key=lambda c: c.name):+ if not category.test_cases:+ continue- overview = assessment.overview+ # Calculate pass/fail/error stats from test casespassing = sum(- result.passing for result in overview.vulnerability_type_results- )- failing = sum(- result.failing for result in overview.vulnerability_type_results+ 1+ for tc in category.test_cases+ if tc.score is not None and tc.score > 0)errored = sum(- result.errored for result in overview.vulnerability_type_results+ 1 for tc in category.test_cases if tc.error is not None)+ failing = len(category.test_cases) - passing - erroredtotal = passing + failingpass_rate = passing / total if total > 0 else 0.0+ # Get vulnerabilities directly from risk_category+ vuln_lines = []vulnerability_groups = defaultdict(list)- for result in overview.vulnerability_type_results:- vulnerability_groups[result.vulnerability].append(- result.vulnerability_type.value- )+ for vuln in category.vulnerabilities:+ vuln_name = vuln.get_name()+ for vuln_type in vuln.get_types():+ vulnerability_groups[vuln_name].append(vuln_type.value)- vuln_lines = []for (vulnerability_name,vulnerability_types,@@ -1023,10 +1087,8 @@ class RedTeamer:vulnerabilty_names = "\n".join(vuln_lines) if vuln_lines else "N/A"- # ----- Attack methods (simple list) ------ attack_list = [- amr.attack_method for amr in overview.attack_method_results- ]+ # Get attacks directly from risk_category+ attack_list = [attack.get_name() for attack in category.attacks]attack_names = "\n".join(attack_list) if attack_list else "N/A"# Color-coded pass rate@@ -1038,7 +1100,7 @@ class RedTeamer:pass_rate_str = f"[bold red]{pass_rate:.0%}[/bold red]"table.add_row(- category_name,+ category.name,pass_rate_str,str(passing),str(failing),diff --git a/deepteam/red_teamer/risk_assessment.py b/deepteam/red_teamer/risk_assessment.pyindex c7ce2058..78655e54 100644--- a/deepteam/red_teamer/risk_assessment.py+++ b/deepteam/red_teamer/risk_assessment.py@@ -1,5 +1,5 @@-from pydantic import BaseModel, Field-from typing import Dict, Optional, List, Any, Union+from pydantic import BaseModel, Field, ConfigDict+from typing import Dict, Optional, List, Any, Union, TYPE_CHECKINGimport datetimeimport osimport json@@ -7,6 +7,7 @@ from enum import Enumfrom deepteam.vulnerabilities.types import VulnerabilityTypefrom deepteam.test_case import RTTestCase+from deepteam.risk_categories.risk_category import RiskCategoryclass TestCasesList(list):@@ -125,8 +126,10 @@ class EnumEncoder(json.JSONEncoder):class RiskAssessment(BaseModel):+ model_config = ConfigDict(arbitrary_types_allowed=True)overview: RedTeamingOverviewtest_cases: List[RTTestCase]+ risk_categories: Optional[List[RiskCategory]] = Field(None)def __init__(self, **data):super().__init__(**data)@@ -255,3 +258,6 @@ def construct_risk_assessment_overview(errored=errored,run_duration=run_duration,)+++RiskAssessment.model_rebuild()diff --git a/deepteam/risk_categories/__init__.py b/deepteam/risk_categories/__init__.pynew file mode 100644index 00000000..f0167510--- /dev/null+++ b/deepteam/risk_categories/__init__.py@@ -0,0 +1,4 @@+from .utils import getRiskCategory+from .risk_category import RiskCategory++__all__ = ["getRiskCategory", "RiskCategory"]diff --git a/deepteam/frameworks/risk_category.py b/deepteam/risk_categories/risk_category.pysimilarity index 75%rename from deepteam/frameworks/risk_category.pyrename to deepteam/risk_categories/risk_category.pyindex b42ba0e3..f05718a1 100644--- a/deepteam/frameworks/risk_category.py+++ b/deepteam/risk_categories/risk_category.py@@ -1,9 +1,11 @@from typing import List, Optional+from dataclasses import dataclass+from deepteam.vulnerabilities import (BaseVulnerability,)from deepteam.attacks import BaseAttack-from dataclasses import dataclass+from deepteam.test_case import RTTestCase@dataclass@@ -11,3 +13,4 @@ class RiskCategory:name: strvulnerabilities: List[BaseVulnerability]attacks: List[BaseAttack]+ test_cases: Optional[List[RTTestCase]] = Nonediff --git a/deepteam/risks/risks.py b/deepteam/risk_categories/utils.pysimilarity index 100%rename from deepteam/risks/risks.pyrename to deepteam/risk_categories/utils.pydiff --git a/deepteam/risks/__init__.py b/deepteam/risks/__init__.pydeleted file mode 100644index 9b0d31eb..00000000--- a/deepteam/risks/__init__.py+++ /dev/null@@ -1 +0,0 @@-from .risks import getRiskCategorydiff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex f735b07d..728af4f3 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -14,7 +14,7 @@ from deepteam.metrics import BFLAMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import BFLATemplateBFLALiteral = Literal[diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex 612532d8..ccca3536 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -13,7 +13,7 @@ from deepteam.metrics import BiasMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import BiasTemplateBiasLiteralType = Literal["religion", "politics", "gender", "race"]diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex 7354505f..4277d7c4 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -14,7 +14,7 @@ from deepteam.metrics import BOLAMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import BOLATemplateBOLALiteral = Literal[diff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.pyindex 1f0aad48..c4e27749 100644--- a/deepteam/vulnerabilities/child_protection/child_protection.py+++ b/deepteam/vulnerabilities/child_protection/child_protection.py@@ -13,7 +13,7 @@ from deepteam.metrics import ChildProtectionMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import ChildProtectionTemplateChildProtectionLiteralType = Literal[diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex cf699bd7..bb1acbff 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -15,7 +15,7 @@ from deepteam.metrics import CompetitorsMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import CompetitionTemplatediff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex ad0f4ae9..13c518f5 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -12,7 +12,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.metrics import BaseRedTeamingMetric, HarmMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom deepteam.test_case import RTTestCasefrom .template import CustomVulnerabilityTemplatediff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex 0cc54998..66d91f7f 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -14,7 +14,7 @@ from deepteam.metrics import DebugAccessMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import DebugAccessTemplateDebugAccessLiteral = Literal[diff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.pyindex 8f16d78c..76ca2e03 100644--- a/deepteam/vulnerabilities/ethics/ethics.py+++ b/deepteam/vulnerabilities/ethics/ethics.py@@ -13,7 +13,7 @@ from deepteam.metrics import EthicsMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import EthicsTemplateEthicsLiteralType = Literal[diff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.pyindex eba5481e..c93f0bb3 100644--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py@@ -53,7 +53,7 @@ class ExcessiveAgency(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[ExcessiveAgencyType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,@@ -108,7 +108,7 @@ class ExcessiveAgency(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[ExcessiveAgencyType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,diff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.pyindex 1d76e1c6..d80dae72 100644--- a/deepteam/vulnerabilities/fairness/fairness.py+++ b/deepteam/vulnerabilities/fairness/fairness.py@@ -13,7 +13,7 @@ from deepteam.metrics import FairnessMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import FairnessTemplateFairnessLiteralType = Literal[diff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.pyindex 2582b178..7c8a5f0f 100644--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py@@ -55,7 +55,7 @@ class GoalTheft(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[GoalTheftType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,@@ -110,7 +110,7 @@ class GoalTheft(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[GoalTheftType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 9ee5a3aa..606db00b 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -14,7 +14,7 @@ from deepteam.metrics import GraphicMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import GraphicContentTemplateGraphicContentLiteral = Literal[diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex e28cc246..ab6af4e2 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -14,7 +14,7 @@ from deepteam.metrics import IllegalMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import IllegalActivityTemplateIllegalActivityLiteral = Literal[diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex 6f34c29f..3b7bd01c 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -16,7 +16,7 @@ from deepteam.metrics import IntellectualPropertyMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import IntellectualPropertyTemplateIntellectualPropertyLiteral = Literal[diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex 12698444..32ad0f92 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -14,7 +14,7 @@ from deepteam.metrics import MisinformationMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import MisinformationTemplatediff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex 8be4d671..ab3aed53 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -14,7 +14,7 @@ from deepteam.metrics import SafetyMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import PersonalSafetyTemplatePersonalSafetyLiteral = Literal[diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex 520097f4..28bd6b76 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -14,7 +14,7 @@ from deepteam.metrics import PIIMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import PIILeakageTemplatePIILeakageLiteral = Literal[diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex 8bf44b2e..d225dc6f 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -14,7 +14,7 @@ from deepteam.metrics import PromptExtractionMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import PromptLeakageTemplatePromptLeakageLiteral = Literal[diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex 233983de..cd977454 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -14,7 +14,7 @@ from deepteam.metrics import RBACMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import RBACTemplateRBACLiteral = Literal[diff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.pyindex 000db93a..f03fa382 100644--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py@@ -59,7 +59,7 @@ class RecursiveHijacking(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[RecursiveHijackingType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,@@ -114,7 +114,7 @@ class RecursiveHijacking(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[RecursiveHijackingType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.pyindex 5095f923..fe2412f0 100644--- a/deepteam/vulnerabilities/robustness/robustness.py+++ b/deepteam/vulnerabilities/robustness/robustness.py@@ -54,7 +54,7 @@ class Robustness(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[RobustnessType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,@@ -109,7 +109,7 @@ class Robustness(BaseVulnerability):model_callback: CallbackType,purpose: Optional[str] = None,) -> Dict[RobustnessType, List[RTTestCase]]:- from deepteam.risks import getRiskCategory+ from deepteam.risk_categories.utils import getRiskCategoryvalidate_model_callback_signature(model_callback=model_callback,diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex 3e1386b9..1648faf0 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -14,7 +14,7 @@ from deepteam.metrics import ShellInjectionMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import ShellInjectionTemplateShellInjectionLiteral = Literal[diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex e888f55e..08436c5f 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -14,7 +14,7 @@ from deepteam.metrics import SQLInjectionMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import SQLInjectionTemplateSQLInjectionLiteral = Literal[diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex e80730ac..e5cdea99 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -14,7 +14,7 @@ from deepteam.metrics import SSRFMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import SSRFTemplateSSRFLiteral = Literal[diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex ef5464bf..192ec958 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -14,7 +14,7 @@ from deepteam.metrics import ToxicityMetric, BaseRedTeamingMetricfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList-from deepteam.risks import getRiskCategory+from deepteam.risk_categories.utils import getRiskCategoryfrom .template import ToxicityTemplateToxicityLiteral = Literal["profanity", "insults", "threats", "mockery"]diff --git a/kk.py b/kk.pyindex 994bab52..9c2bc0e6 100644--- a/kk.py+++ b/kk.py@@ -2,6 +2,7 @@ from deepteam import red_teamfrom deepteam.vulnerabilities import Bias, BOLAfrom deepteam.attacks.single_turn import Roleplayimport os+from deepteam.frameworks import OWASPTop10async def your_callback(input: str) -> str:@@ -11,9 +12,10 @@ async def your_callback(input: str) -> str:print(os.environ.get("CONFIDENT_API_KEY"))risk_assessment = red_team(- attacks=[Roleplay()],- vulnerabilities=[Bias(), BOLA()],+ # attacks=[Roleplay()],+ # vulnerabilities=[Bias(), BOLA()],model_callback=your_callback,+ framework=OWASPTop10(),)from deepteam.vulnerabilities.constants import (diff --git a/tests/test_frameworks/test_mitre.py b/tests/test_frameworks/test_mitre.pyindex ebefc393..f46bae89 100644--- a/tests/test_frameworks/test_mitre.py+++ b/tests/test_frameworks/test_mitre.py@@ -2,7 +2,7 @@ import pytestfrom deepteam.frameworks import MITREfrom deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.attacks import BaseAttack-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam.frameworks.mitre.risk_categories import MITRE_CATEGORIESfrom deepteam import red_teamdiff --git a/tests/test_frameworks/test_nist.py b/tests/test_frameworks/test_nist.pyindex 833efe99..44f1301a 100644--- a/tests/test_frameworks/test_nist.py+++ b/tests/test_frameworks/test_nist.py@@ -3,7 +3,7 @@ from deepteam.frameworks import NISTfrom deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.attacks import BaseAttackfrom deepteam.frameworks.nist.risk_categories import NIST_CATEGORIES-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam import red_teamdiff --git a/tests/test_frameworks/test_owasp_top_10.py b/tests/test_frameworks/test_owasp_top_10.pyindex 1b98f4ff..18e1603e 100644--- a/tests/test_frameworks/test_owasp_top_10.py+++ b/tests/test_frameworks/test_owasp_top_10.py@@ -3,7 +3,7 @@ from deepteam.frameworks import OWASPTop10from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.attacks import BaseAttackfrom deepteam.frameworks.owasp.risk_categories import OWASP_CATEGORIES-from deepteam.frameworks.risk_category import RiskCategory+from deepteam.risk_categories import RiskCategoryfrom deepteam import red_team