DeepTeam is a framework to red team LLMs and AI agents.
.
sid-muralid66b55bbb5f8
diff --git a/deepteam/__init__.py b/deepteam/__init__.pyindex 07e6092e..6b972894 100644--- a/deepteam/__init__.py+++ b/deepteam/__init__.py@@ -4,9 +4,11 @@ import re# Optionally add telemetryfrom deepteam.red_team import red_team+from deepteam.frameworks import FrameworkType__all__ = ["red_team",+ "FrameworkType",]diff --git a/deepteam/attacks/__init__.py b/deepteam/attacks/__init__.pyindex bded7292..c896415d 100644--- a/deepteam/attacks/__init__.py+++ b/deepteam/attacks/__init__.py@@ -1,15 +1,8 @@# from .attack_simulator.attack_simulator import AttackSimulator, Attackfrom .base_attack import BaseAttack-# from .single_turn import *-# from .multi_turn import *--# Generic enhancement attacks (moved to single_turn)-from .single_turn.authority_spoofing.authority_spoofing import AuthoritySpoofing-from .single_turn.role_manipulation.role_manipulation import RoleManipulation-from .single_turn.goal_redirection.goal_redirection import GoalRedirection-from .single_turn.linguistic_confusion.linguistic_confusion import LinguisticConfusion-from .single_turn.validation_bypass.validation_bypass import ValidationBypass-from .single_turn.context_injection.context_injection import ContextInjection+# Import all single_turn and multi_turn attacks+from .single_turn import *+from .multi_turn import *# All agentic-specific attacks have been embedded into their corresponding vulnerabilitiesdiff --git a/deepteam/attacks/attack_simulator/template.py b/deepteam/attacks/attack_simulator/template.pyindex 6388228c..7fa38540 100644--- a/deepteam/attacks/attack_simulator/template.py+++ b/deepteam/attacks/attack_simulator/template.py@@ -38,12 +38,12 @@ from deepteam.vulnerabilities.custom.custom_types import CustomVulnerabilityTypefrom deepteam.vulnerabilities.custom.template import CustomVulnerabilityTemplate# Import remaining agentic vulnerability types and templates-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.types import RecursiveGoalSubversionType-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.template import RecursiveGoalSubversionTemplate-from deepteam.vulnerabilities.agentic.goal_extraction.types import GoalExtractionType-from deepteam.vulnerabilities.agentic.goal_extraction.template import GoalExtractionTemplate-from deepteam.vulnerabilities.agentic.output_verification.types import OutputVerificationType-from deepteam.vulnerabilities.agentic.output_verification.template import OutputVerificationTemplate+from deepteam.vulnerabilities.agentic.recursive_hijacking.types import RecursiveHijackingType+from deepteam.vulnerabilities.agentic.recursive_hijacking.template import RecursiveHijackingTemplate+from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType+from deepteam.vulnerabilities.agentic.goal_theft.template import GoalTheftTemplate+from deepteam.vulnerabilities.agentic.validation_bypass.types import ValidationBypassType+from deepteam.vulnerabilities.agentic.validation_bypass.template import ValidationBypassTemplateTEMPLATE_MAP: Dict[Type[VulnerabilityType], TemplateType] = {BiasType: BiasTemplate,@@ -60,9 +60,9 @@ TEMPLATE_MAP: Dict[Type[VulnerabilityType], TemplateType] = {ToxicityType: ToxicityTemplate,UnauthorizedAccessType: UnauthorizedAccessTemplate,# Remaining agentic vulnerability types- RecursiveGoalSubversionType: RecursiveGoalSubversionTemplate,- GoalExtractionType: GoalExtractionTemplate,- OutputVerificationType: OutputVerificationTemplate,+ RecursiveHijackingType: RecursiveHijackingTemplate,+ GoalTheftType: GoalTheftTemplate,+ ValidationBypassType: ValidationBypassTemplate,}diff --git a/deepteam/attacks/single_turn/__init__.py b/deepteam/attacks/single_turn/__init__.pyindex 4a2dcc2b..7b9d8179 100644--- a/deepteam/attacks/single_turn/__init__.py+++ b/deepteam/attacks/single_turn/__init__.py@@ -11,7 +11,7 @@ from .rot13 import ROT13# Generic enhancement attacks (moved from agentic)from .system_override.system_override import SystemOverridefrom .permission_escalation.permission_escalation import PermissionEscalation-from .objective_reframing.objective_reframing import ObjectiveReframing-from .semantic_manipulation.semantic_manipulation import SemanticManipulation+from .objective_reframing.objective_reframing import GoalRedirection+from .semantic_manipulation.semantic_manipulation import LinguisticConfusionfrom .input_bypass.input_bypass import InputBypassfrom .context_poisoning.context_poisoning import ContextPoisoningdiff --git a/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py b/deepteam/attacks/single_turn/objective_reframing/objective_reframing.pyindex ad19ebdb..e9b72ff0 100644--- a/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py+++ b/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py@@ -1,7 +1,7 @@import randomfrom deepteam.attacks import BaseAttack-from deepteam.attacks.single_turn.goal_redirection.template import GoalRedirectionTemplate+from deepteam.attacks.single_turn.objective_reframing.template import GoalRedirectionTemplateclass GoalRedirection(BaseAttack):diff --git a/deepteam/attacks/single_turn/permission_escalation/__init__.py b/deepteam/attacks/single_turn/permission_escalation/__init__.pyindex 7452f0df..03fc8214 100644--- a/deepteam/attacks/single_turn/permission_escalation/__init__.py+++ b/deepteam/attacks/single_turn/permission_escalation/__init__.py@@ -1 +1 @@-from .template import RoleManipulationTemplate\ No newline at end of file+from .template import PermissionEscalationTemplate\ No newline at end of filediff --git a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.pyindex 888d1976..1e7a9fa4 100644--- a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py+++ b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py@@ -1,7 +1,7 @@import randomfrom deepteam.attacks import BaseAttack-from deepteam.attacks.single_turn.linguistic_confusion.template import LinguisticConfusionTemplate+from deepteam.attacks.single_turn.semantic_manipulation.template import LinguisticConfusionTemplateclass LinguisticConfusion(BaseAttack):diff --git a/deepteam/frameworks/__init__.py b/deepteam/frameworks/__init__.pynew file mode 100644index 00000000..ac7f7aec--- /dev/null+++ b/deepteam/frameworks/__init__.py@@ -0,0 +1,3 @@+from .frameworks import get_framework_config, get_combined_framework_config, FrameworkType++__all__ = ["get_framework_config", "get_combined_framework_config", "FrameworkType"]\ No newline at end of filediff --git a/deepteam/frameworks/frameworks.py b/deepteam/frameworks/frameworks.pynew file mode 100644index 00000000..f771f4b0--- /dev/null+++ b/deepteam/frameworks/frameworks.py@@ -0,0 +1,302 @@+from enum import Enum+from typing import List, Dict, Any, Union+from dataclasses import dataclass++from deepteam.vulnerabilities import (+ BaseVulnerability,+ Bias,+ Toxicity,+ Misinformation,+ IllegalActivity,+ PromptLeakage,+ PIILeakage,+ UnauthorizedAccess,+ ExcessiveAgency,+ Robustness,+ IntellectualProperty,+ Competition,+ GraphicContent,+ PersonalSafety,+ RBAC,+ BOLA,+ BFLA,+ SSRF,+ DebugAccess,+ ShellInjection,+ SQLInjection,+)++# Import agentic vulnerabilities+from deepteam.vulnerabilities.agentic import (+ GoalTheft,+ ValidationBypass,+ RecursiveHijacking,+)++from deepteam.attacks import BaseAttack+from deepteam.attacks.single_turn import (+ Base64,+ GrayBox,+ Leetspeak,+ MathProblem,+ Multilingual,+ PromptInjection,+ PromptProbing,+ Roleplay,+ ROT13,+ SystemOverride,+ PermissionEscalation,+ GoalRedirection,+ LinguisticConfusion,+ InputBypass,+ ContextPoisoning,+)++from deepteam.attacks.multi_turn import (+ CrescendoJailbreaking,+ LinearJailbreaking,+ TreeJailbreaking,+ SequentialJailbreak,+ BadLikertJudge,+)+++class FrameworkType(Enum):+ OWASP = "owasp"+ AGENTIC = "agentic"+++@dataclass+class FrameworkConfig:+ name: str+ vulnerabilities: List[BaseVulnerability]+ attacks: List[BaseAttack]+++def _get_owasp_vulnerabilities() -> List[BaseVulnerability]:+ """+ Get vulnerabilities based on OWASP Top 10 for LLMs 2025.++ Reference: docs/red-teaming-owasp-top-10-for-llms.mdx+ """+ return [+ # 1. Prompt Injection - Handled by attacks, not vulnerabilities++ # 2. Sensitive Information Disclosure+ PIILeakage(types=["direct disclosure", "api and database access", "session leak", "social manipulation"]),+ PromptLeakage(types=["secrets and credentials", "instructions", "guard exposure", "permissions and roles"]),+ IntellectualProperty(types=["imitation", "copyright violations", "trademark infringement", "patent disclosure"]),++ # 3. Supply Chain - Not directly testable via attacks++ # 4. Data and Model Poisoning+ Bias(types=["race", "gender", "religion", "politics"]),+ Toxicity(types=["profanity", "insults", "threats", "mockery"]),+ Misinformation(types=["factual errors", "unsupported claims", "expertize misrepresentation"]),+ IllegalActivity(types=["cybercrime", "violent crimes", "non violent crimes", "illegal drugs"]),+ GraphicContent(types=["sexual content", "graphic content", "pornographic content"]),+ PersonalSafety(types=["bullying", "self-harm", "unsafe practices", "dangerous challenges"]),++ # 5. Improper Output Handling - Not directly testable via attacks++ # 6. Excessive Agency+ ExcessiveAgency(types=["functionality", "permissions", "autonomy"]),++ # 7. System Prompt Leakage (covered under Sensitive Information Disclosure)++ # 8. Vector and Embedding Weaknesses - Not directly testable via attacks++ # 9. Misinformation (already covered above)+ Competition(types=["discreditation"]),++ # 10. Unbounded Consumption - Not directly testable via attacks++ # Additional OWASP-related vulnerabilities+ RBAC(types=["role bypass", "privilege escalation", "unauthorized role assumption"]),+ DebugAccess(types=["debug_mode_bypass", "development_endpoint_access", "administrative_interface_exposure"]),+ ShellInjection(types=["command_injection", "system_command_execution", "shell_escape_sequences"]),+ SQLInjection(types=["blind_sql_injection", "union_based_injection", "error_based_injection"]),+ BFLA(types=["privilege_escalation", "function_bypass", "authorization_bypass"]),+ BOLA(types=["object_access_bypass", "cross_customer_access", "unauthorized_object_manipulation"]),+ SSRF(types=["internal_service_access", "cloud_metadata_access", "port_scanning"]),+ Robustness(types=["hijacking", "input overreliance"]),+ ]+++def _get_owasp_attacks() -> List[BaseAttack]:+ """+ Get attacks relevant for OWASP Top 10 for LLMs testing.+ """+ return [+ # Core injection attacks+ PromptInjection(weight=3),++ # Encoding-based attacks for bypassing filters+ Base64(weight=2),+ ROT13(weight=2),+ Leetspeak(weight=2),++ # Social engineering attacks+ Roleplay(weight=2),+ PromptProbing(weight=2),++ # Multilingual and obfuscation attacks+ Multilingual(weight=2),+ MathProblem(weight=1),++ # Gray box attacks for known vulnerabilities+ GrayBox(weight=2),++ # Multi-turn sophisticated attacks+ CrescendoJailbreaking(weight=2),+ LinearJailbreaking(weight=2),+ TreeJailbreaking(weight=1),+ SequentialJailbreak(weight=2),+ BadLikertJudge(weight=1),+ ]+++def _get_agentic_vulnerabilities() -> List[BaseVulnerability]:+ """+ Get vulnerabilities specific to agentic AI systems.+ """+ return [+ # Core agentic vulnerabilities+ GoalTheft(types=["escalating_probing", "cooperative_dialogue", "social_engineering"]),+ ValidationBypass(types=["unverified_claims", "false_confidence", "misleading_responses"]),+ RecursiveHijacking(types=["self_modifying_goals", "recursive_objective_chaining", "goal_propagation_attacks"]),++ # Excessive agency and robustness for agentic systems+ ExcessiveAgency(types=["functionality", "permissions", "autonomy"]),+ Robustness(types=["hijacking", "input overreliance"]),++ # Access control vulnerabilities relevant to agents+ RBAC(types=["role bypass", "privilege escalation", "unauthorized role assumption"]),+ BFLA(types=["privilege_escalation", "function_bypass", "authorization_bypass"]),+ BOLA(types=["object_access_bypass", "cross_customer_access", "unauthorized_object_manipulation"]),++ # Information leakage vulnerabilities for agents+ PIILeakage(types=["session leak", "social manipulation"]),+ PromptLeakage(types=["instructions", "guard exposure"]),++ # Output quality vulnerabilities for agents+ Misinformation(types=["factual errors", "unsupported claims"]),+ Bias(types=["race", "gender", "politics"]),+ ]+++def _get_agentic_attacks() -> List[BaseAttack]:+ """+ Get attacks specifically designed for agentic AI systems.+ """+ return [+ # Core agentic attack methods+ SystemOverride(weight=3),+ PermissionEscalation(weight=3),+ GoalRedirection(weight=3),+ LinguisticConfusion(weight=2),+ InputBypass(weight=2),+ ContextPoisoning(weight=3),++ # Enhanced prompt injection for agents+ PromptInjection(weight=2),++ # Social engineering attacks for agents+ Roleplay(weight=2),+ PromptProbing(weight=2),++ # Multi-turn attacks for persistent agents+ CrescendoJailbreaking(weight=2),+ LinearJailbreaking(weight=2),+ TreeJailbreaking(weight=2),+ SequentialJailbreak(weight=3),+ BadLikertJudge(weight=2),++ # Obfuscation attacks+ Base64(weight=1),+ Multilingual(weight=1),+ GrayBox(weight=1),+ ]+++def get_framework_config(framework: Union[str, FrameworkType]) -> FrameworkConfig:+ """+ Get the configuration for a specific framework.++ Args:+ framework: Either a string ("owasp", "agentic") or FrameworkType enum++ Returns:+ FrameworkConfig containing vulnerabilities and attacks for the framework++ Raises:+ ValueError: If framework is not supported+ """+ if isinstance(framework, str):+ try:+ framework = FrameworkType(framework.lower())+ except ValueError:+ raise ValueError(f"Unsupported framework: {framework}. Supported frameworks: {[f.value for f in FrameworkType]}")++ if framework == FrameworkType.OWASP:+ return FrameworkConfig(+ name="OWASP Top 10 for LLMs",+ vulnerabilities=_get_owasp_vulnerabilities(),+ attacks=_get_owasp_attacks()+ )+ elif framework == FrameworkType.AGENTIC:+ return FrameworkConfig(+ name="Agentic AI Security Framework",+ vulnerabilities=_get_agentic_vulnerabilities(),+ attacks=_get_agentic_attacks()+ )+ else:+ raise ValueError(f"Unsupported framework: {framework}")+++def get_combined_framework_config(frameworks: List[Union[str, FrameworkType]]) -> FrameworkConfig:+ """+ Get combined configuration for multiple frameworks.++ Args:+ frameworks: List of framework names or FrameworkType enums++ Returns:+ FrameworkConfig containing merged vulnerabilities and attacks+ """+ if not frameworks:+ raise ValueError("At least one framework must be specified")++ all_vulnerabilities = []+ all_attacks = []+ framework_names = []++ # Collect vulnerabilities and attacks from all frameworks+ for framework in frameworks:+ config = get_framework_config(framework)+ framework_names.append(config.name)+ all_vulnerabilities.extend(config.vulnerabilities)+ all_attacks.extend(config.attacks)++ # Remove duplicates while preserving order+ seen_vuln_names = set()+ unique_vulnerabilities = []+ for vuln in all_vulnerabilities:+ vuln_name = vuln.get_name()+ if vuln_name not in seen_vuln_names:+ seen_vuln_names.add(vuln_name)+ unique_vulnerabilities.append(vuln)++ seen_attack_names = set()+ unique_attacks = []+ for attack in all_attacks:+ attack_name = attack.get_name()+ if attack_name not in seen_attack_names:+ seen_attack_names.add(attack_name)+ unique_attacks.append(attack)++ return FrameworkConfig(+ name=f"Combined Framework ({', '.join(framework_names)})",+ vulnerabilities=unique_vulnerabilities,+ attacks=unique_attacks+ )\ No newline at end of filediff --git a/deepteam/red_team.py b/deepteam/red_team.pyindex cc3a63c8..09bc4de6 100644--- a/deepteam/red_team.py+++ b/deepteam/red_team.py@@ -1,15 +1,17 @@-from typing import List, Optional+from typing import List, Optional, Unionfrom deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.attacks import BaseAttackfrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.red_teamer import RedTeamer+from deepteam.frameworks import get_combined_framework_config, FrameworkTypedef red_team(model_callback: CallbackType,- vulnerabilities: List[BaseVulnerability],- attacks: List[BaseAttack],+ vulnerabilities: Optional[List[BaseVulnerability]] = None,+ attacks: Optional[List[BaseAttack]] = None,+ frameworks: Optional[List[Union[str, FrameworkType]]] = None,attacks_per_vulnerability_type: int = 1,ignore_errors: bool = False,run_async: bool = True,@@ -17,6 +19,17 @@ def red_team(target_purpose: Optional[str] = None,):+ if frameworks:+ framework_config = get_combined_framework_config(frameworks)+ final_vulnerabilities = framework_config.vulnerabilities+ final_attacks = framework_config.attacks+ else:+ if not vulnerabilities or not attacks:+ raise ValueError("Either 'frameworks' must be provided, or both 'vulnerabilities' and 'attacks' must be provided.")++ final_vulnerabilities = vulnerabilities+ final_attacks = attacks+red_teamer = RedTeamer(async_mode=run_async,max_concurrent=max_concurrent,@@ -24,8 +37,8 @@ def red_team()risk_assessment = red_teamer.red_team(model_callback=model_callback,- vulnerabilities=vulnerabilities,- attacks=attacks,+ vulnerabilities=final_vulnerabilities,+ attacks=final_attacks,attacks_per_vulnerability_type=attacks_per_vulnerability_type,ignore_errors=ignore_errors,)diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex 513f5c9e..73863270 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -29,13 +29,20 @@ from deepteam.vulnerabilities.types import (PIILeakageType,ToxicityType,BiasType,+ RBACType,+ BOLAType,+ BFLAType,+ SSRFType,+ DebugAccessType,+ ShellInjectionType,+ SQLInjectionType,VulnerabilityType,)# Import agentic vulnerability types-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.types import RecursiveGoalSubversionType-from deepteam.vulnerabilities.agentic.goal_extraction.types import GoalExtractionType-from deepteam.vulnerabilities.agentic.output_verification.types import OutputVerificationType+from deepteam.vulnerabilities.agentic.recursive_hijacking.types import RecursiveHijackingType+from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType+from deepteam.vulnerabilities.agentic.validation_bypass.types import ValidationBypassTypefrom deepteam.vulnerabilities.agentic.robustness.types import RobustnessTypefrom deepteam.vulnerabilities.agentic.excessive_agency.types import ExcessiveAgencyType@@ -480,35 +487,62 @@ class RedTeamer:)for pii_type in PIILeakageType},- #### Unauthorized Access ####- UnauthorizedAccessType.DEBUG_ACCESS: lambda: DebugAccessMetric(- model=self.evaluation_model, async_mode=self.async_mode- ),- UnauthorizedAccessType.RBAC: lambda: RBACMetric(- model=self.evaluation_model,- purpose=self.target_purpose,- async_mode=self.async_mode,- ),- UnauthorizedAccessType.SHELL_INJECTION: lambda: ShellInjectionMetric(- model=self.evaluation_model, async_mode=self.async_mode- ),- UnauthorizedAccessType.SQL_INJECTION: lambda: SQLInjectionMetric(- model=self.evaluation_model, async_mode=self.async_mode- ),- UnauthorizedAccessType.BFLA: lambda: BFLAMetric(- purpose=self.target_purpose,- model=self.evaluation_model,- async_mode=self.async_mode,- ),- UnauthorizedAccessType.BOLA: lambda: BOLAMetric(- model=self.evaluation_model,- async_mode=self.async_mode,- ),- UnauthorizedAccessType.SSRF: lambda: SSRFMetric(- purpose=self.target_purpose,- model=self.evaluation_model,- async_mode=self.async_mode,- ),+ #### Debug Access ####+ **{+ debug_access_type: lambda: DebugAccessMetric(+ model=self.evaluation_model, async_mode=self.async_mode+ )+ for debug_access_type in DebugAccessType+ },+ #### Shell Injection ####+ **{+ shell_injection_type: lambda: ShellInjectionMetric(+ model=self.evaluation_model, async_mode=self.async_mode+ )+ for shell_injection_type in ShellInjectionType+ },+ #### SQL Injection ####+ **{+ sql_injection_type: lambda: SQLInjectionMetric(+ model=self.evaluation_model, async_mode=self.async_mode+ )+ for sql_injection_type in SQLInjectionType+ },+ #### BFLA ####+ **{+ bfla_type: lambda: BFLAMetric(+ purpose=self.target_purpose,+ model=self.evaluation_model,+ async_mode=self.async_mode,+ )+ for bfla_type in BFLAType+ },+ #### BOLA ####+ **{+ bola_type: lambda: BOLAMetric(+ model=self.evaluation_model,+ async_mode=self.async_mode,+ )+ for bola_type in BOLAType+ },+ #### SSRF ####+ **{+ ssrf_type: lambda: SSRFMetric(+ purpose=self.target_purpose,+ model=self.evaluation_model,+ async_mode=self.async_mode,+ )+ for ssrf_type in SSRFType+ },+ #### RBAC ####+ **{+ rbac_type: lambda: RBACMetric(+ model=self.evaluation_model,+ purpose=self.target_purpose,+ async_mode=self.async_mode,+ )+ for rbac_type in RBACType+ },#### Excessive Agency ####**{excessive_agency_type: lambda: ExcessiveAgencyMetric(@@ -572,7 +606,7 @@ class RedTeamer:purpose=self.target_purpose,async_mode=self.async_mode,)- for subversion_type in RecursiveGoalSubversionType+ for subversion_type in RecursiveHijackingType},**{extraction_type: lambda: ExtractionSuccessMetric(@@ -580,7 +614,7 @@ class RedTeamer:purpose=self.target_purpose,async_mode=self.async_mode,)- for extraction_type in GoalExtractionType+ for extraction_type in GoalTheftType},**{verification_type: lambda: VerificationAssessmentMetric(@@ -588,7 +622,7 @@ class RedTeamer:purpose=self.target_purpose,async_mode=self.async_mode,)- for verification_type in OutputVerificationType+ for verification_type in ValidationBypassType},}diff --git a/deepteam/vulnerabilities/__init__.py b/deepteam/vulnerabilities/__init__.pyindex eec454e4..4fcd4346 100644--- a/deepteam/vulnerabilities/__init__.py+++ b/deepteam/vulnerabilities/__init__.py@@ -12,6 +12,7 @@ from .debug_access.debug_access import DebugAccessfrom .shell_injection.shell_injection import ShellInjectionfrom .sql_injection.sql_injection import SQLInjectionfrom .ssrf.ssrf import SSRF+from .unauthorized_access.unauthorized_access import UnauthorizedAccessfrom .intellectual_property.intellectual_property import IntellectualPropertyfrom .competition.competition import Competitionfrom .graphic_content.graphic_content import GraphicContentdiff --git a/deepteam/vulnerabilities/rbac/types.py b/deepteam/vulnerabilities/rbac/types.pyindex 0519ecba..20f2fc79 100644--- a/deepteam/vulnerabilities/rbac/types.py+++ b/deepteam/vulnerabilities/rbac/types.py@@ -1 +1,15 @@-\ No newline at end of file+from enum import Enum+from typing import Literal+++class RBACType(Enum):+ ROLE_BYPASS = "role bypass"+ PRIVILEGE_ESCALATION = "privilege escalation"+ UNAUTHORIZED_ROLE_ASSUMPTION = "unauthorized role assumption"+++RBACTypes = Literal[+ RBACType.ROLE_BYPASS.value,+ RBACType.PRIVILEGE_ESCALATION.value,+ RBACType.UNAUTHORIZED_ROLE_ASSUMPTION.value,+]\ No newline at end of filediff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.pydeleted file mode 100644index 6261a9a5..00000000--- a/deepteam/vulnerabilities/robustness/robustness.py+++ /dev/null@@ -1,23 +0,0 @@-from typing import List, Literal, Optional--from deepteam.vulnerabilities import BaseVulnerability-from deepteam.vulnerabilities.robustness import RobustnessType-from deepteam.vulnerabilities.utils import validate_vulnerability_types--RobustnessLiteral = Literal["input overreliance", "hijacking"]---class Robustness(BaseVulnerability):- def __init__(- self,- types: Optional[List[RobustnessLiteral]] = [- type.value for type in RobustnessType- ],- ):- enum_types = validate_vulnerability_types(- self.get_name(), types=types, allowed_type=RobustnessType- )- super().__init__(types=enum_types)-- def get_name(self) -> str:- return "Robustness"\ No newline at end of filediff --git a/deepteam/vulnerabilities/types.py b/deepteam/vulnerabilities/types.pyindex 48e08286..75e92041 100644--- a/deepteam/vulnerabilities/types.py+++ b/deepteam/vulnerabilities/types.py@@ -36,25 +36,35 @@ from deepteam.vulnerabilities.pii_leakage import PIILeakageTypefrom deepteam.vulnerabilities.toxicity import ToxicityTypefrom deepteam.vulnerabilities.bias import BiasTypefrom deepteam.vulnerabilities.custom import CustomVulnerabilityType+from deepteam.vulnerabilities.rbac import RBACType+from deepteam.vulnerabilities.bola.types import BOLAType+from deepteam.vulnerabilities.bfla.types import BFLAType+from deepteam.vulnerabilities.ssrf.types import SSRFType+from deepteam.vulnerabilities.debug_access.types import DebugAccessType+from deepteam.vulnerabilities.shell_injection.types import ShellInjectionType+from deepteam.vulnerabilities.sql_injection.types import SQLInjectionTypefrom deepteam.vulnerabilities.unauthorized_access.template import (UnauthorizedAccessTemplate,)--# Import restored vulnerability types and templates+from deepteam.vulnerabilities.rbac.template import RBACTemplate+from deepteam.vulnerabilities.bola.template import BOLATemplate+from deepteam.vulnerabilities.bfla.template import BFLATemplate+from deepteam.vulnerabilities.ssrf.template import SSRFTemplate+from deepteam.vulnerabilities.debug_access.template import DebugAccessTemplate+from deepteam.vulnerabilities.shell_injection.template import ShellInjectionTemplate+from deepteam.vulnerabilities.sql_injection.template import SQLInjectionTemplatefrom deepteam.vulnerabilities.agentic.robustness import RobustnessTypefrom deepteam.vulnerabilities.agentic.robustness.template import RobustnessTemplatefrom deepteam.vulnerabilities.agentic.excessive_agency import ExcessiveAgencyTypefrom deepteam.vulnerabilities.agentic.excessive_agency.template import ExcessiveAgencyTemplate# Import agentic vulnerability types-from deepteam.vulnerabilities.agentic.goal_extraction.types import GoalExtractionType-from deepteam.vulnerabilities.agentic.output_verification.types import OutputVerificationType-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.types import RecursiveGoalSubversionType--# Import agentic vulnerability templates-from deepteam.vulnerabilities.agentic.goal_extraction.template import GoalExtractionTemplate-from deepteam.vulnerabilities.agentic.output_verification.template import OutputVerificationTemplate-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.template import RecursiveGoalSubversionTemplate+from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType+from deepteam.vulnerabilities.agentic.validation_bypass.types import ValidationBypassType+from deepteam.vulnerabilities.agentic.recursive_hijacking.types import RecursiveHijackingType+from deepteam.vulnerabilities.agentic.goal_theft.template import GoalTheftTemplate+from deepteam.vulnerabilities.agentic.validation_bypass.template import ValidationBypassTemplate+from deepteam.vulnerabilities.agentic.recursive_hijacking.template import RecursiveHijackingTemplateVulnerabilityType = Union[UnauthorizedAccessType,@@ -72,13 +82,20 @@ VulnerabilityType = Union[IntellectualPropertyType,IntellectualPropertyType,CustomVulnerabilityType,+ RBACType,+ BOLAType,+ BFLAType,+ SSRFType,+ DebugAccessType,+ ShellInjectionType,+ SQLInjectionType,# Restored vulnerability typesRobustnessType,ExcessiveAgencyType,# Agentic vulnerability types- GoalExtractionType,- OutputVerificationType,- RecursiveGoalSubversionType,+ GoalTheftType,+ ValidationBypassType,+ RecursiveHijackingType,]TemplateType = Union[@@ -93,10 +110,16 @@ TemplateType = Union[PromptLeakageTemplate,ToxicityTemplate,UnauthorizedAccessTemplate,+ RBACTemplate,+ BOLATemplate,+ BFLATemplate,+ SSRFTemplate,+ DebugAccessTemplate,+ ShellInjectionTemplate,+ SQLInjectionTemplate,RobustnessTemplate,ExcessiveAgencyTemplate,- # Agentic vulnerability templates- GoalExtractionTemplate,- OutputVerificationTemplate,- RecursiveGoalSubversionTemplate,+ GoalTheftTemplate,+ ValidationBypassTemplate,+ RecursiveHijackingTemplate,]diff --git a/tests/test_frameworks.py b/tests/test_frameworks.pynew file mode 100644index 00000000..34735b3d--- /dev/null+++ b/tests/test_frameworks.py@@ -0,0 +1,101 @@+#!/usr/bin/env python3++"""+Simple test script for framework functionality.+Tests 4 scenarios: both frameworks, individual frameworks, manual vul/attacks, and error case.+"""++import asyncio+from deepteam import red_team, FrameworkType+from deepteam.vulnerabilities import Bias+from deepteam.attacks.single_turn import PromptInjection++async def test_model_callback(input: str) -> str:+ """Simple test model callback."""+ return f"I cannot help with that request: {input}"++def test_frameworks():+ """Test framework functionality with 4 different scenarios."""++ print("π§ͺ Testing DeepTeam Framework Functionality")+ print("=" * 50)++ # Test 1: Both frameworks at once+ print("\n1οΈβ£ Testing both frameworks together...")+ try:+ result = red_team(+ model_callback=test_model_callback,+ frameworks=["owasp", "agentic"],+ attacks_per_vulnerability_type=1,+ ignore_errors=True+ )+ print(f"β Both frameworks: {len(result.test_cases)} test cases")+ except Exception as e:+ print(f"β Both frameworks failed: {e}")+ return False++ # Test 2a: OWASP framework only+ print("\n2οΈβ£ Testing OWASP framework only...")+ try:+ result = red_team(+ model_callback=test_model_callback,+ frameworks=["owasp"],+ attacks_per_vulnerability_type=1,+ ignore_errors=True+ )+ print(f"β OWASP framework: {len(result.test_cases)} test cases")+ except Exception as e:+ print(f"β OWASP framework failed: {e}")+ return False++ # Test 2b: Agentic framework only+ print("\n3οΈβ£ Testing Agentic framework only...")+ try:+ result = red_team(+ model_callback=test_model_callback,+ frameworks=[FrameworkType.AGENTIC],+ attacks_per_vulnerability_type=1,+ ignore_errors=True+ )+ print(f"β Agentic framework: {len(result.test_cases)} test cases")+ except Exception as e:+ print(f"β Agentic framework failed: {e}")+ return False++ # Test 3: Manual vulnerabilities and attacks (original API)+ print("\n4οΈβ£ Testing manual vulnerabilities and attacks...")+ try:+ result = red_team(+ model_callback=test_model_callback,+ vulnerabilities=[Bias(types=["race"])],+ attacks=[PromptInjection()],+ attacks_per_vulnerability_type=1,+ ignore_errors=True+ )+ print(f"β Manual vul/attacks: {len(result.test_cases)} test cases")+ except Exception as e:+ print(f"β Manual vul/attacks failed: {e}")+ return False++ # Test 4: No framework, no vulnerabilities, no attacks (should fail)+ print("\n5οΈβ£ Testing error case (no frameworks, no vul/attacks)...")+ try:+ result = red_team(+ model_callback=test_model_callback,+ attacks_per_vulnerability_type=1,+ ignore_errors=True+ )+ print("β Error case should have failed but didn't")+ return False+ except ValueError as e:+ print(f"β Error case correctly failed: {e}")+ except Exception as e:+ print(f"β Error case failed with wrong exception: {e}")+ return False++ print("\nπ All framework tests passed!")+ return True++if __name__ == "__main__":+ success = test_frameworks()+ exit(0 if success else 1)\ No newline at end of file