Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

.

sid-muralid66b55bbb5f8
diff --git a/deepteam/__init__.py b/deepteam/__init__.py
index 07e6092e..6b972894 100644
--- a/deepteam/__init__.py
+++ b/deepteam/__init__.py
@@ -4,9 +4,11 @@ import re
# Optionally add telemetry
from deepteam.red_team import red_team
+from deepteam.frameworks import FrameworkType
__all__ = [
"red_team",
+ "FrameworkType",
]
diff --git a/deepteam/attacks/__init__.py b/deepteam/attacks/__init__.py
index bded7292..c896415d 100644
--- a/deepteam/attacks/__init__.py
+++ b/deepteam/attacks/__init__.py
@@ -1,15 +1,8 @@
# from .attack_simulator.attack_simulator import AttackSimulator, Attack
from .base_attack import BaseAttack
-# from .single_turn import *
-# from .multi_turn import *
-
-# Generic enhancement attacks (moved to single_turn)
-from .single_turn.authority_spoofing.authority_spoofing import AuthoritySpoofing
-from .single_turn.role_manipulation.role_manipulation import RoleManipulation
-from .single_turn.goal_redirection.goal_redirection import GoalRedirection
-from .single_turn.linguistic_confusion.linguistic_confusion import LinguisticConfusion
-from .single_turn.validation_bypass.validation_bypass import ValidationBypass
-from .single_turn.context_injection.context_injection import ContextInjection
+# Import all single_turn and multi_turn attacks
+from .single_turn import *
+from .multi_turn import *
# All agentic-specific attacks have been embedded into their corresponding vulnerabilities
diff --git a/deepteam/attacks/attack_simulator/template.py b/deepteam/attacks/attack_simulator/template.py
index 6388228c..7fa38540 100644
--- a/deepteam/attacks/attack_simulator/template.py
+++ b/deepteam/attacks/attack_simulator/template.py
@@ -38,12 +38,12 @@ from deepteam.vulnerabilities.custom.custom_types import CustomVulnerabilityType
from deepteam.vulnerabilities.custom.template import CustomVulnerabilityTemplate
# Import remaining agentic vulnerability types and templates
-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.types import RecursiveGoalSubversionType
-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.template import RecursiveGoalSubversionTemplate
-from deepteam.vulnerabilities.agentic.goal_extraction.types import GoalExtractionType
-from deepteam.vulnerabilities.agentic.goal_extraction.template import GoalExtractionTemplate
-from deepteam.vulnerabilities.agentic.output_verification.types import OutputVerificationType
-from deepteam.vulnerabilities.agentic.output_verification.template import OutputVerificationTemplate
+from deepteam.vulnerabilities.agentic.recursive_hijacking.types import RecursiveHijackingType
+from deepteam.vulnerabilities.agentic.recursive_hijacking.template import RecursiveHijackingTemplate
+from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType
+from deepteam.vulnerabilities.agentic.goal_theft.template import GoalTheftTemplate
+from deepteam.vulnerabilities.agentic.validation_bypass.types import ValidationBypassType
+from deepteam.vulnerabilities.agentic.validation_bypass.template import ValidationBypassTemplate
TEMPLATE_MAP: Dict[Type[VulnerabilityType], TemplateType] = {
BiasType: BiasTemplate,
@@ -60,9 +60,9 @@ TEMPLATE_MAP: Dict[Type[VulnerabilityType], TemplateType] = {
ToxicityType: ToxicityTemplate,
UnauthorizedAccessType: UnauthorizedAccessTemplate,
# Remaining agentic vulnerability types
- RecursiveGoalSubversionType: RecursiveGoalSubversionTemplate,
- GoalExtractionType: GoalExtractionTemplate,
- OutputVerificationType: OutputVerificationTemplate,
+ RecursiveHijackingType: RecursiveHijackingTemplate,
+ GoalTheftType: GoalTheftTemplate,
+ ValidationBypassType: ValidationBypassTemplate,
}
diff --git a/deepteam/attacks/single_turn/__init__.py b/deepteam/attacks/single_turn/__init__.py
index 4a2dcc2b..7b9d8179 100644
--- a/deepteam/attacks/single_turn/__init__.py
+++ b/deepteam/attacks/single_turn/__init__.py
@@ -11,7 +11,7 @@ from .rot13 import ROT13
# Generic enhancement attacks (moved from agentic)
from .system_override.system_override import SystemOverride
from .permission_escalation.permission_escalation import PermissionEscalation
-from .objective_reframing.objective_reframing import ObjectiveReframing
-from .semantic_manipulation.semantic_manipulation import SemanticManipulation
+from .objective_reframing.objective_reframing import GoalRedirection
+from .semantic_manipulation.semantic_manipulation import LinguisticConfusion
from .input_bypass.input_bypass import InputBypass
from .context_poisoning.context_poisoning import ContextPoisoning
diff --git a/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py b/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py
index ad19ebdb..e9b72ff0 100644
--- a/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py
+++ b/deepteam/attacks/single_turn/objective_reframing/objective_reframing.py
@@ -1,7 +1,7 @@
import random
from deepteam.attacks import BaseAttack
-from deepteam.attacks.single_turn.goal_redirection.template import GoalRedirectionTemplate
+from deepteam.attacks.single_turn.objective_reframing.template import GoalRedirectionTemplate
class GoalRedirection(BaseAttack):
diff --git a/deepteam/attacks/single_turn/permission_escalation/__init__.py b/deepteam/attacks/single_turn/permission_escalation/__init__.py
index 7452f0df..03fc8214 100644
--- a/deepteam/attacks/single_turn/permission_escalation/__init__.py
+++ b/deepteam/attacks/single_turn/permission_escalation/__init__.py
@@ -1 +1 @@
-from .template import RoleManipulationTemplate
\ No newline at end of file
+from .template import PermissionEscalationTemplate
\ No newline at end of file
diff --git a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py
index 888d1976..1e7a9fa4 100644
--- a/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py
+++ b/deepteam/attacks/single_turn/semantic_manipulation/semantic_manipulation.py
@@ -1,7 +1,7 @@
import random
from deepteam.attacks import BaseAttack
-from deepteam.attacks.single_turn.linguistic_confusion.template import LinguisticConfusionTemplate
+from deepteam.attacks.single_turn.semantic_manipulation.template import LinguisticConfusionTemplate
class LinguisticConfusion(BaseAttack):
diff --git a/deepteam/frameworks/__init__.py b/deepteam/frameworks/__init__.py
new file mode 100644
index 00000000..ac7f7aec
--- /dev/null
+++ b/deepteam/frameworks/__init__.py
@@ -0,0 +1,3 @@
+from .frameworks import get_framework_config, get_combined_framework_config, FrameworkType
+
+__all__ = ["get_framework_config", "get_combined_framework_config", "FrameworkType"]
\ No newline at end of file
diff --git a/deepteam/frameworks/frameworks.py b/deepteam/frameworks/frameworks.py
new file mode 100644
index 00000000..f771f4b0
--- /dev/null
+++ b/deepteam/frameworks/frameworks.py
@@ -0,0 +1,302 @@
+from enum import Enum
+from typing import List, Dict, Any, Union
+from dataclasses import dataclass
+
+from deepteam.vulnerabilities import (
+ BaseVulnerability,
+ Bias,
+ Toxicity,
+ Misinformation,
+ IllegalActivity,
+ PromptLeakage,
+ PIILeakage,
+ UnauthorizedAccess,
+ ExcessiveAgency,
+ Robustness,
+ IntellectualProperty,
+ Competition,
+ GraphicContent,
+ PersonalSafety,
+ RBAC,
+ BOLA,
+ BFLA,
+ SSRF,
+ DebugAccess,
+ ShellInjection,
+ SQLInjection,
+)
+
+# Import agentic vulnerabilities
+from deepteam.vulnerabilities.agentic import (
+ GoalTheft,
+ ValidationBypass,
+ RecursiveHijacking,
+)
+
+from deepteam.attacks import BaseAttack
+from deepteam.attacks.single_turn import (
+ Base64,
+ GrayBox,
+ Leetspeak,
+ MathProblem,
+ Multilingual,
+ PromptInjection,
+ PromptProbing,
+ Roleplay,
+ ROT13,
+ SystemOverride,
+ PermissionEscalation,
+ GoalRedirection,
+ LinguisticConfusion,
+ InputBypass,
+ ContextPoisoning,
+)
+
+from deepteam.attacks.multi_turn import (
+ CrescendoJailbreaking,
+ LinearJailbreaking,
+ TreeJailbreaking,
+ SequentialJailbreak,
+ BadLikertJudge,
+)
+
+
+class FrameworkType(Enum):
+ OWASP = "owasp"
+ AGENTIC = "agentic"
+
+
+@dataclass
+class FrameworkConfig:
+ name: str
+ vulnerabilities: List[BaseVulnerability]
+ attacks: List[BaseAttack]
+
+
+def _get_owasp_vulnerabilities() -> List[BaseVulnerability]:
+ """
+ Get vulnerabilities based on OWASP Top 10 for LLMs 2025.
+
+ Reference: docs/red-teaming-owasp-top-10-for-llms.mdx
+ """
+ return [
+ # 1. Prompt Injection - Handled by attacks, not vulnerabilities
+
+ # 2. Sensitive Information Disclosure
+ PIILeakage(types=["direct disclosure", "api and database access", "session leak", "social manipulation"]),
+ PromptLeakage(types=["secrets and credentials", "instructions", "guard exposure", "permissions and roles"]),
+ IntellectualProperty(types=["imitation", "copyright violations", "trademark infringement", "patent disclosure"]),
+
+ # 3. Supply Chain - Not directly testable via attacks
+
+ # 4. Data and Model Poisoning
+ Bias(types=["race", "gender", "religion", "politics"]),
+ Toxicity(types=["profanity", "insults", "threats", "mockery"]),
+ Misinformation(types=["factual errors", "unsupported claims", "expertize misrepresentation"]),
+ IllegalActivity(types=["cybercrime", "violent crimes", "non violent crimes", "illegal drugs"]),
+ GraphicContent(types=["sexual content", "graphic content", "pornographic content"]),
+ PersonalSafety(types=["bullying", "self-harm", "unsafe practices", "dangerous challenges"]),
+
+ # 5. Improper Output Handling - Not directly testable via attacks
+
+ # 6. Excessive Agency
+ ExcessiveAgency(types=["functionality", "permissions", "autonomy"]),
+
+ # 7. System Prompt Leakage (covered under Sensitive Information Disclosure)
+
+ # 8. Vector and Embedding Weaknesses - Not directly testable via attacks
+
+ # 9. Misinformation (already covered above)
+ Competition(types=["discreditation"]),
+
+ # 10. Unbounded Consumption - Not directly testable via attacks
+
+ # Additional OWASP-related vulnerabilities
+ RBAC(types=["role bypass", "privilege escalation", "unauthorized role assumption"]),
+ DebugAccess(types=["debug_mode_bypass", "development_endpoint_access", "administrative_interface_exposure"]),
+ ShellInjection(types=["command_injection", "system_command_execution", "shell_escape_sequences"]),
+ SQLInjection(types=["blind_sql_injection", "union_based_injection", "error_based_injection"]),
+ BFLA(types=["privilege_escalation", "function_bypass", "authorization_bypass"]),
+ BOLA(types=["object_access_bypass", "cross_customer_access", "unauthorized_object_manipulation"]),
+ SSRF(types=["internal_service_access", "cloud_metadata_access", "port_scanning"]),
+ Robustness(types=["hijacking", "input overreliance"]),
+ ]
+
+
+def _get_owasp_attacks() -> List[BaseAttack]:
+ """
+ Get attacks relevant for OWASP Top 10 for LLMs testing.
+ """
+ return [
+ # Core injection attacks
+ PromptInjection(weight=3),
+
+ # Encoding-based attacks for bypassing filters
+ Base64(weight=2),
+ ROT13(weight=2),
+ Leetspeak(weight=2),
+
+ # Social engineering attacks
+ Roleplay(weight=2),
+ PromptProbing(weight=2),
+
+ # Multilingual and obfuscation attacks
+ Multilingual(weight=2),
+ MathProblem(weight=1),
+
+ # Gray box attacks for known vulnerabilities
+ GrayBox(weight=2),
+
+ # Multi-turn sophisticated attacks
+ CrescendoJailbreaking(weight=2),
+ LinearJailbreaking(weight=2),
+ TreeJailbreaking(weight=1),
+ SequentialJailbreak(weight=2),
+ BadLikertJudge(weight=1),
+ ]
+
+
+def _get_agentic_vulnerabilities() -> List[BaseVulnerability]:
+ """
+ Get vulnerabilities specific to agentic AI systems.
+ """
+ return [
+ # Core agentic vulnerabilities
+ GoalTheft(types=["escalating_probing", "cooperative_dialogue", "social_engineering"]),
+ ValidationBypass(types=["unverified_claims", "false_confidence", "misleading_responses"]),
+ RecursiveHijacking(types=["self_modifying_goals", "recursive_objective_chaining", "goal_propagation_attacks"]),
+
+ # Excessive agency and robustness for agentic systems
+ ExcessiveAgency(types=["functionality", "permissions", "autonomy"]),
+ Robustness(types=["hijacking", "input overreliance"]),
+
+ # Access control vulnerabilities relevant to agents
+ RBAC(types=["role bypass", "privilege escalation", "unauthorized role assumption"]),
+ BFLA(types=["privilege_escalation", "function_bypass", "authorization_bypass"]),
+ BOLA(types=["object_access_bypass", "cross_customer_access", "unauthorized_object_manipulation"]),
+
+ # Information leakage vulnerabilities for agents
+ PIILeakage(types=["session leak", "social manipulation"]),
+ PromptLeakage(types=["instructions", "guard exposure"]),
+
+ # Output quality vulnerabilities for agents
+ Misinformation(types=["factual errors", "unsupported claims"]),
+ Bias(types=["race", "gender", "politics"]),
+ ]
+
+
+def _get_agentic_attacks() -> List[BaseAttack]:
+ """
+ Get attacks specifically designed for agentic AI systems.
+ """
+ return [
+ # Core agentic attack methods
+ SystemOverride(weight=3),
+ PermissionEscalation(weight=3),
+ GoalRedirection(weight=3),
+ LinguisticConfusion(weight=2),
+ InputBypass(weight=2),
+ ContextPoisoning(weight=3),
+
+ # Enhanced prompt injection for agents
+ PromptInjection(weight=2),
+
+ # Social engineering attacks for agents
+ Roleplay(weight=2),
+ PromptProbing(weight=2),
+
+ # Multi-turn attacks for persistent agents
+ CrescendoJailbreaking(weight=2),
+ LinearJailbreaking(weight=2),
+ TreeJailbreaking(weight=2),
+ SequentialJailbreak(weight=3),
+ BadLikertJudge(weight=2),
+
+ # Obfuscation attacks
+ Base64(weight=1),
+ Multilingual(weight=1),
+ GrayBox(weight=1),
+ ]
+
+
+def get_framework_config(framework: Union[str, FrameworkType]) -> FrameworkConfig:
+ """
+ Get the configuration for a specific framework.
+
+ Args:
+ framework: Either a string ("owasp", "agentic") or FrameworkType enum
+
+ Returns:
+ FrameworkConfig containing vulnerabilities and attacks for the framework
+
+ Raises:
+ ValueError: If framework is not supported
+ """
+ if isinstance(framework, str):
+ try:
+ framework = FrameworkType(framework.lower())
+ except ValueError:
+ raise ValueError(f"Unsupported framework: {framework}. Supported frameworks: {[f.value for f in FrameworkType]}")
+
+ if framework == FrameworkType.OWASP:
+ return FrameworkConfig(
+ name="OWASP Top 10 for LLMs",
+ vulnerabilities=_get_owasp_vulnerabilities(),
+ attacks=_get_owasp_attacks()
+ )
+ elif framework == FrameworkType.AGENTIC:
+ return FrameworkConfig(
+ name="Agentic AI Security Framework",
+ vulnerabilities=_get_agentic_vulnerabilities(),
+ attacks=_get_agentic_attacks()
+ )
+ else:
+ raise ValueError(f"Unsupported framework: {framework}")
+
+
+def get_combined_framework_config(frameworks: List[Union[str, FrameworkType]]) -> FrameworkConfig:
+ """
+ Get combined configuration for multiple frameworks.
+
+ Args:
+ frameworks: List of framework names or FrameworkType enums
+
+ Returns:
+ FrameworkConfig containing merged vulnerabilities and attacks
+ """
+ if not frameworks:
+ raise ValueError("At least one framework must be specified")
+
+ all_vulnerabilities = []
+ all_attacks = []
+ framework_names = []
+
+ # Collect vulnerabilities and attacks from all frameworks
+ for framework in frameworks:
+ config = get_framework_config(framework)
+ framework_names.append(config.name)
+ all_vulnerabilities.extend(config.vulnerabilities)
+ all_attacks.extend(config.attacks)
+
+ # Remove duplicates while preserving order
+ seen_vuln_names = set()
+ unique_vulnerabilities = []
+ for vuln in all_vulnerabilities:
+ vuln_name = vuln.get_name()
+ if vuln_name not in seen_vuln_names:
+ seen_vuln_names.add(vuln_name)
+ unique_vulnerabilities.append(vuln)
+
+ seen_attack_names = set()
+ unique_attacks = []
+ for attack in all_attacks:
+ attack_name = attack.get_name()
+ if attack_name not in seen_attack_names:
+ seen_attack_names.add(attack_name)
+ unique_attacks.append(attack)
+
+ return FrameworkConfig(
+ name=f"Combined Framework ({', '.join(framework_names)})",
+ vulnerabilities=unique_vulnerabilities,
+ attacks=unique_attacks
+ )
\ No newline at end of file
diff --git a/deepteam/red_team.py b/deepteam/red_team.py
index cc3a63c8..09bc4de6 100644
--- a/deepteam/red_team.py
+++ b/deepteam/red_team.py
@@ -1,15 +1,17 @@
-from typing import List, Optional
+from typing import List, Optional, Union
from deepteam.vulnerabilities import BaseVulnerability
from deepteam.attacks import BaseAttack
from deepteam.attacks.multi_turn.types import CallbackType
from deepteam.red_teamer import RedTeamer
+from deepteam.frameworks import get_combined_framework_config, FrameworkType
def red_team(
model_callback: CallbackType,
- vulnerabilities: List[BaseVulnerability],
- attacks: List[BaseAttack],
+ vulnerabilities: Optional[List[BaseVulnerability]] = None,
+ attacks: Optional[List[BaseAttack]] = None,
+ frameworks: Optional[List[Union[str, FrameworkType]]] = None,
attacks_per_vulnerability_type: int = 1,
ignore_errors: bool = False,
run_async: bool = True,
@@ -17,6 +19,17 @@ def red_team(
target_purpose: Optional[str] = None,
):
+ if frameworks:
+ framework_config = get_combined_framework_config(frameworks)
+ final_vulnerabilities = framework_config.vulnerabilities
+ final_attacks = framework_config.attacks
+ else:
+ if not vulnerabilities or not attacks:
+ raise ValueError("Either 'frameworks' must be provided, or both 'vulnerabilities' and 'attacks' must be provided.")
+
+ final_vulnerabilities = vulnerabilities
+ final_attacks = attacks
+
red_teamer = RedTeamer(
async_mode=run_async,
max_concurrent=max_concurrent,
@@ -24,8 +37,8 @@ def red_team(
)
risk_assessment = red_teamer.red_team(
model_callback=model_callback,
- vulnerabilities=vulnerabilities,
- attacks=attacks,
+ vulnerabilities=final_vulnerabilities,
+ attacks=final_attacks,
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
ignore_errors=ignore_errors,
)
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index 513f5c9e..73863270 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -29,13 +29,20 @@ from deepteam.vulnerabilities.types import (
PIILeakageType,
ToxicityType,
BiasType,
+ RBACType,
+ BOLAType,
+ BFLAType,
+ SSRFType,
+ DebugAccessType,
+ ShellInjectionType,
+ SQLInjectionType,
VulnerabilityType,
)
# Import agentic vulnerability types
-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.types import RecursiveGoalSubversionType
-from deepteam.vulnerabilities.agentic.goal_extraction.types import GoalExtractionType
-from deepteam.vulnerabilities.agentic.output_verification.types import OutputVerificationType
+from deepteam.vulnerabilities.agentic.recursive_hijacking.types import RecursiveHijackingType
+from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType
+from deepteam.vulnerabilities.agentic.validation_bypass.types import ValidationBypassType
from deepteam.vulnerabilities.agentic.robustness.types import RobustnessType
from deepteam.vulnerabilities.agentic.excessive_agency.types import ExcessiveAgencyType
@@ -480,35 +487,62 @@ class RedTeamer:
)
for pii_type in PIILeakageType
},
- #### Unauthorized Access ####
- UnauthorizedAccessType.DEBUG_ACCESS: lambda: DebugAccessMetric(
- model=self.evaluation_model, async_mode=self.async_mode
- ),
- UnauthorizedAccessType.RBAC: lambda: RBACMetric(
- model=self.evaluation_model,
- purpose=self.target_purpose,
- async_mode=self.async_mode,
- ),
- UnauthorizedAccessType.SHELL_INJECTION: lambda: ShellInjectionMetric(
- model=self.evaluation_model, async_mode=self.async_mode
- ),
- UnauthorizedAccessType.SQL_INJECTION: lambda: SQLInjectionMetric(
- model=self.evaluation_model, async_mode=self.async_mode
- ),
- UnauthorizedAccessType.BFLA: lambda: BFLAMetric(
- purpose=self.target_purpose,
- model=self.evaluation_model,
- async_mode=self.async_mode,
- ),
- UnauthorizedAccessType.BOLA: lambda: BOLAMetric(
- model=self.evaluation_model,
- async_mode=self.async_mode,
- ),
- UnauthorizedAccessType.SSRF: lambda: SSRFMetric(
- purpose=self.target_purpose,
- model=self.evaluation_model,
- async_mode=self.async_mode,
- ),
+ #### Debug Access ####
+ **{
+ debug_access_type: lambda: DebugAccessMetric(
+ model=self.evaluation_model, async_mode=self.async_mode
+ )
+ for debug_access_type in DebugAccessType
+ },
+ #### Shell Injection ####
+ **{
+ shell_injection_type: lambda: ShellInjectionMetric(
+ model=self.evaluation_model, async_mode=self.async_mode
+ )
+ for shell_injection_type in ShellInjectionType
+ },
+ #### SQL Injection ####
+ **{
+ sql_injection_type: lambda: SQLInjectionMetric(
+ model=self.evaluation_model, async_mode=self.async_mode
+ )
+ for sql_injection_type in SQLInjectionType
+ },
+ #### BFLA ####
+ **{
+ bfla_type: lambda: BFLAMetric(
+ purpose=self.target_purpose,
+ model=self.evaluation_model,
+ async_mode=self.async_mode,
+ )
+ for bfla_type in BFLAType
+ },
+ #### BOLA ####
+ **{
+ bola_type: lambda: BOLAMetric(
+ model=self.evaluation_model,
+ async_mode=self.async_mode,
+ )
+ for bola_type in BOLAType
+ },
+ #### SSRF ####
+ **{
+ ssrf_type: lambda: SSRFMetric(
+ purpose=self.target_purpose,
+ model=self.evaluation_model,
+ async_mode=self.async_mode,
+ )
+ for ssrf_type in SSRFType
+ },
+ #### RBAC ####
+ **{
+ rbac_type: lambda: RBACMetric(
+ model=self.evaluation_model,
+ purpose=self.target_purpose,
+ async_mode=self.async_mode,
+ )
+ for rbac_type in RBACType
+ },
#### Excessive Agency ####
**{
excessive_agency_type: lambda: ExcessiveAgencyMetric(
@@ -572,7 +606,7 @@ class RedTeamer:
purpose=self.target_purpose,
async_mode=self.async_mode,
)
- for subversion_type in RecursiveGoalSubversionType
+ for subversion_type in RecursiveHijackingType
},
**{
extraction_type: lambda: ExtractionSuccessMetric(
@@ -580,7 +614,7 @@ class RedTeamer:
purpose=self.target_purpose,
async_mode=self.async_mode,
)
- for extraction_type in GoalExtractionType
+ for extraction_type in GoalTheftType
},
**{
verification_type: lambda: VerificationAssessmentMetric(
@@ -588,7 +622,7 @@ class RedTeamer:
purpose=self.target_purpose,
async_mode=self.async_mode,
)
- for verification_type in OutputVerificationType
+ for verification_type in ValidationBypassType
},
}
diff --git a/deepteam/vulnerabilities/__init__.py b/deepteam/vulnerabilities/__init__.py
index eec454e4..4fcd4346 100644
--- a/deepteam/vulnerabilities/__init__.py
+++ b/deepteam/vulnerabilities/__init__.py
@@ -12,6 +12,7 @@ from .debug_access.debug_access import DebugAccess
from .shell_injection.shell_injection import ShellInjection
from .sql_injection.sql_injection import SQLInjection
from .ssrf.ssrf import SSRF
+from .unauthorized_access.unauthorized_access import UnauthorizedAccess
from .intellectual_property.intellectual_property import IntellectualProperty
from .competition.competition import Competition
from .graphic_content.graphic_content import GraphicContent
diff --git a/deepteam/vulnerabilities/rbac/types.py b/deepteam/vulnerabilities/rbac/types.py
index 0519ecba..20f2fc79 100644
--- a/deepteam/vulnerabilities/rbac/types.py
+++ b/deepteam/vulnerabilities/rbac/types.py
@@ -1 +1,15 @@
-
\ No newline at end of file
+from enum import Enum
+from typing import Literal
+
+
+class RBACType(Enum):
+ ROLE_BYPASS = "role bypass"
+ PRIVILEGE_ESCALATION = "privilege escalation"
+ UNAUTHORIZED_ROLE_ASSUMPTION = "unauthorized role assumption"
+
+
+RBACTypes = Literal[
+ RBACType.ROLE_BYPASS.value,
+ RBACType.PRIVILEGE_ESCALATION.value,
+ RBACType.UNAUTHORIZED_ROLE_ASSUMPTION.value,
+]
\ No newline at end of file
diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.py
deleted file mode 100644
index 6261a9a5..00000000
--- a/deepteam/vulnerabilities/robustness/robustness.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from typing import List, Literal, Optional
-
-from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.vulnerabilities.robustness import RobustnessType
-from deepteam.vulnerabilities.utils import validate_vulnerability_types
-
-RobustnessLiteral = Literal["input overreliance", "hijacking"]
-
-
-class Robustness(BaseVulnerability):
- def __init__(
- self,
- types: Optional[List[RobustnessLiteral]] = [
- type.value for type in RobustnessType
- ],
- ):
- enum_types = validate_vulnerability_types(
- self.get_name(), types=types, allowed_type=RobustnessType
- )
- super().__init__(types=enum_types)
-
- def get_name(self) -> str:
- return "Robustness"
\ No newline at end of file
diff --git a/deepteam/vulnerabilities/types.py b/deepteam/vulnerabilities/types.py
index 48e08286..75e92041 100644
--- a/deepteam/vulnerabilities/types.py
+++ b/deepteam/vulnerabilities/types.py
@@ -36,25 +36,35 @@ from deepteam.vulnerabilities.pii_leakage import PIILeakageType
from deepteam.vulnerabilities.toxicity import ToxicityType
from deepteam.vulnerabilities.bias import BiasType
from deepteam.vulnerabilities.custom import CustomVulnerabilityType
+from deepteam.vulnerabilities.rbac import RBACType
+from deepteam.vulnerabilities.bola.types import BOLAType
+from deepteam.vulnerabilities.bfla.types import BFLAType
+from deepteam.vulnerabilities.ssrf.types import SSRFType
+from deepteam.vulnerabilities.debug_access.types import DebugAccessType
+from deepteam.vulnerabilities.shell_injection.types import ShellInjectionType
+from deepteam.vulnerabilities.sql_injection.types import SQLInjectionType
from deepteam.vulnerabilities.unauthorized_access.template import (
UnauthorizedAccessTemplate,
)
-
-# Import restored vulnerability types and templates
+from deepteam.vulnerabilities.rbac.template import RBACTemplate
+from deepteam.vulnerabilities.bola.template import BOLATemplate
+from deepteam.vulnerabilities.bfla.template import BFLATemplate
+from deepteam.vulnerabilities.ssrf.template import SSRFTemplate
+from deepteam.vulnerabilities.debug_access.template import DebugAccessTemplate
+from deepteam.vulnerabilities.shell_injection.template import ShellInjectionTemplate
+from deepteam.vulnerabilities.sql_injection.template import SQLInjectionTemplate
from deepteam.vulnerabilities.agentic.robustness import RobustnessType
from deepteam.vulnerabilities.agentic.robustness.template import RobustnessTemplate
from deepteam.vulnerabilities.agentic.excessive_agency import ExcessiveAgencyType
from deepteam.vulnerabilities.agentic.excessive_agency.template import ExcessiveAgencyTemplate
# Import agentic vulnerability types
-from deepteam.vulnerabilities.agentic.goal_extraction.types import GoalExtractionType
-from deepteam.vulnerabilities.agentic.output_verification.types import OutputVerificationType
-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.types import RecursiveGoalSubversionType
-
-# Import agentic vulnerability templates
-from deepteam.vulnerabilities.agentic.goal_extraction.template import GoalExtractionTemplate
-from deepteam.vulnerabilities.agentic.output_verification.template import OutputVerificationTemplate
-from deepteam.vulnerabilities.agentic.recursive_goal_subversion.template import RecursiveGoalSubversionTemplate
+from deepteam.vulnerabilities.agentic.goal_theft.types import GoalTheftType
+from deepteam.vulnerabilities.agentic.validation_bypass.types import ValidationBypassType
+from deepteam.vulnerabilities.agentic.recursive_hijacking.types import RecursiveHijackingType
+from deepteam.vulnerabilities.agentic.goal_theft.template import GoalTheftTemplate
+from deepteam.vulnerabilities.agentic.validation_bypass.template import ValidationBypassTemplate
+from deepteam.vulnerabilities.agentic.recursive_hijacking.template import RecursiveHijackingTemplate
VulnerabilityType = Union[
UnauthorizedAccessType,
@@ -72,13 +82,20 @@ VulnerabilityType = Union[
IntellectualPropertyType,
IntellectualPropertyType,
CustomVulnerabilityType,
+ RBACType,
+ BOLAType,
+ BFLAType,
+ SSRFType,
+ DebugAccessType,
+ ShellInjectionType,
+ SQLInjectionType,
# Restored vulnerability types
RobustnessType,
ExcessiveAgencyType,
# Agentic vulnerability types
- GoalExtractionType,
- OutputVerificationType,
- RecursiveGoalSubversionType,
+ GoalTheftType,
+ ValidationBypassType,
+ RecursiveHijackingType,
]
TemplateType = Union[
@@ -93,10 +110,16 @@ TemplateType = Union[
PromptLeakageTemplate,
ToxicityTemplate,
UnauthorizedAccessTemplate,
+ RBACTemplate,
+ BOLATemplate,
+ BFLATemplate,
+ SSRFTemplate,
+ DebugAccessTemplate,
+ ShellInjectionTemplate,
+ SQLInjectionTemplate,
RobustnessTemplate,
ExcessiveAgencyTemplate,
- # Agentic vulnerability templates
- GoalExtractionTemplate,
- OutputVerificationTemplate,
- RecursiveGoalSubversionTemplate,
+ GoalTheftTemplate,
+ ValidationBypassTemplate,
+ RecursiveHijackingTemplate,
]
diff --git a/tests/test_frameworks.py b/tests/test_frameworks.py
new file mode 100644
index 00000000..34735b3d
--- /dev/null
+++ b/tests/test_frameworks.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+"""
+Simple test script for framework functionality.
+Tests 4 scenarios: both frameworks, individual frameworks, manual vul/attacks, and error case.
+"""
+
+import asyncio
+from deepteam import red_team, FrameworkType
+from deepteam.vulnerabilities import Bias
+from deepteam.attacks.single_turn import PromptInjection
+
+async def test_model_callback(input: str) -> str:
+ """Simple test model callback."""
+ return f"I cannot help with that request: {input}"
+
+def test_frameworks():
+ """Test framework functionality with 4 different scenarios."""
+
+ print("πŸ§ͺ Testing DeepTeam Framework Functionality")
+ print("=" * 50)
+
+ # Test 1: Both frameworks at once
+ print("\n1️⃣ Testing both frameworks together...")
+ try:
+ result = red_team(
+ model_callback=test_model_callback,
+ frameworks=["owasp", "agentic"],
+ attacks_per_vulnerability_type=1,
+ ignore_errors=True
+ )
+ print(f"βœ… Both frameworks: {len(result.test_cases)} test cases")
+ except Exception as e:
+ print(f"❌ Both frameworks failed: {e}")
+ return False
+
+ # Test 2a: OWASP framework only
+ print("\n2️⃣ Testing OWASP framework only...")
+ try:
+ result = red_team(
+ model_callback=test_model_callback,
+ frameworks=["owasp"],
+ attacks_per_vulnerability_type=1,
+ ignore_errors=True
+ )
+ print(f"βœ… OWASP framework: {len(result.test_cases)} test cases")
+ except Exception as e:
+ print(f"❌ OWASP framework failed: {e}")
+ return False
+
+ # Test 2b: Agentic framework only
+ print("\n3️⃣ Testing Agentic framework only...")
+ try:
+ result = red_team(
+ model_callback=test_model_callback,
+ frameworks=[FrameworkType.AGENTIC],
+ attacks_per_vulnerability_type=1,
+ ignore_errors=True
+ )
+ print(f"βœ… Agentic framework: {len(result.test_cases)} test cases")
+ except Exception as e:
+ print(f"❌ Agentic framework failed: {e}")
+ return False
+
+ # Test 3: Manual vulnerabilities and attacks (original API)
+ print("\n4️⃣ Testing manual vulnerabilities and attacks...")
+ try:
+ result = red_team(
+ model_callback=test_model_callback,
+ vulnerabilities=[Bias(types=["race"])],
+ attacks=[PromptInjection()],
+ attacks_per_vulnerability_type=1,
+ ignore_errors=True
+ )
+ print(f"βœ… Manual vul/attacks: {len(result.test_cases)} test cases")
+ except Exception as e:
+ print(f"❌ Manual vul/attacks failed: {e}")
+ return False
+
+ # Test 4: No framework, no vulnerabilities, no attacks (should fail)
+ print("\n5️⃣ Testing error case (no frameworks, no vul/attacks)...")
+ try:
+ result = red_team(
+ model_callback=test_model_callback,
+ attacks_per_vulnerability_type=1,
+ ignore_errors=True
+ )
+ print("❌ Error case should have failed but didn't")
+ return False
+ except ValueError as e:
+ print(f"βœ… Error case correctly failed: {e}")
+ except Exception as e:
+ print(f"❌ Error case failed with wrong exception: {e}")
+ return False
+
+ print("\nπŸŽ‰ All framework tests passed!")
+ return True
+
+if __name__ == "__main__":
+ success = test_frameworks()
+ exit(0 if success else 1)
\ No newline at end of file