DeepTeam is a framework to red team LLMs and AI agents.
Merge branch 'main' into custom-vulnerability
Jeffrey Ip145cf5aa0966
diff --cc b.pyindex 39d38ce2,4122525c..23ec9e0c--- a/b.py+++ b/b.py@@@ -459,280 -148,266 +463,531 @@@ VULNERABILITY_TYPE_MAPPING ="debug_access": UnauthorizedAccessType.DEBUG_ACCESS,"shell_injection": UnauthorizedAccessType.SHELL_INJECTION,"sql_injection": UnauthorizedAccessType.SQL_INJECTION,- "ssrf": UnauthorizedAccessType.SSRF- }+ "ssrf": UnauthorizedAccessType.SSRF,+ },}+class ModelType(Enum):+ GPT35 = "gpt-3.5-turbo"+ GPT4o = "gpt-4o"+ CLAUDE = "claude-3-7-sonnet-latest"++++@dataclass+class TestResult:+ model_name: str+ vulnerability_name: str+ vulnerability_type: str+ success: bool+ error: Optional[str] = None+ is_non_compliant: Optional[bool] = None+ generated_attack: Optional[str] = None+ execution_time: float = 0.0+ def get_all_vulnerability_types() -> str:+ """Return a formatted string listing all available vulnerability types and subtypes."""+ result = "Available vulnerability types and subtypes:\n\n"+ for vul_type, subtypes in VULNERABILITY_TYPE_MAPPING.items():+ result += f"{vul_type}:\n"+ for subtype in subtypes:+ result += f" - {subtype}\n"+ result += "\n"+ return result+++ def get_vulnerability_from_args(+ vul_type: str, subtype: str+ ) -> Optional[VulnerabilityType]:+ """Convert command line arguments to a vulnerability type enum."""+ if vul_type not in VULNERABILITY_TYPE_MAPPING:+ print(f"Error: '{vul_type}' is not a valid vulnerability type.")+ return None++ if subtype not in VULNERABILITY_TYPE_MAPPING[vul_type]:+ print(f"Error: '{subtype}' is not a valid subtype for {vul_type}.")+ print(f"Available subtypes for {vul_type}:")+ for available_subtype in VULNERABILITY_TYPE_MAPPING[vul_type]:+ print(f" - {available_subtype}")+ return None++ return VULNERABILITY_TYPE_MAPPING[vul_type][subtype]++class VulnerabilityTester:+ def __init__(self, purpose: str, model_type: ModelType):+ self.purpose = purpose+ self.model_type = model_type++ # Initialize the appropriate model+ if model_type in [ModelType.GPT35, ModelType.GPT4o]:+ self.model = GPTModel(model_name=model_type.value)+ elif model_type == ModelType.CLAUDE:+ self.model = AnthropicModel(model=model_type.value)++ self.simulator = AttackSimulator(+ purpose=purpose,+ max_concurrent=3,+ simulator_model=self.model+ )++ def test_vulnerability(self, vulnerability_class, vulnerability_type) -> TestResult:+ start_time = time.time()+ try:+ # Create vulnerability instance+ vulnerability = vulnerability_class(types=[vulnerability_type.value])++ # Generate baseline attacks+ try:+ attacks = self.simulator.simulate_baseline_attacks(+ attacks_per_vulnerability_type=1,+ vulnerability=vulnerability,+ ignore_errors=True+ )+ except Exception as e:+ return TestResult(+ model_name=self.model_type.value,+ vulnerability_name=vulnerability.get_name(),+ vulnerability_type=vulnerability_type.value,+ success=False,+ error=f"Failed to generate attacks: {str(e)}",+ execution_time=time.time() - start_time+ )++ if not attacks or not attacks[0].input:+ return TestResult(+ model_name=self.model_type.value,+ vulnerability_name=vulnerability.get_name(),+ vulnerability_type=vulnerability_type.value,+ success=False,+ error="No attacks generated",+ execution_time=time.time() - start_time+ )++ try:+ non_compliant_prompt = AttackSimulatorTemplate.non_compliant(attacks[0].input)+ non_compliant_response = self.model.generate(non_compliant_prompt)++ if isinstance(non_compliant_response, tuple):+ non_compliant_response = non_compliant_response[0]++ is_non_compliant = ("non_compliant" in non_compliant_response.lower() and "true" in non_compliant_response.lower())+ except Exception as e:+ is_non_compliant = False++ return TestResult(+ model_name=self.model_type.value,+ vulnerability_name=vulnerability.get_name(),+ vulnerability_type=vulnerability_type.value,+ success=True,+ generated_attack=attacks[0].input,+ is_non_compliant=is_non_compliant,+ execution_time=time.time() - start_time+ )++ except Exception as e:+ return TestResult(+ model_name=self.model_type.value,+ vulnerability_name=vulnerability_class.__name__,+ vulnerability_type=vulnerability_type.value,+ success=False,+ error=str(e),+ execution_time=time.time() - start_time+ )-def separator():- """Print a separator line for better readability."""- print("\n" + "=" * 80 + "\n")++def run_systematic_tests():+ """Run tests for all combinations of models and vulnerability types."""+ results = []+ timing_data = []++ # Define models in order+ models = [ModelType.GPT35, ModelType.GPT4o, ModelType.CLAUDE]++ # Map vulnerability type names to their classes for special cases+ SPECIAL_CLASS_MAP = {+ "pii_leakage": PIILeakage,+ "prompt_leakage": PromptLeakage,+ "robustness": Robustness,+ "toxicity": Toxicity,+ "unauthorized_access": UnauthorizedAccess+ }++ # Test each vulnerability type and subtype+ for vuln_type, subtypes in VULNERABILITY_TYPE_MAPPING.items():+ print(f"\n{'='*80}")+ print(f"π Testing {vuln_type}")+ print(f"{'='*80}")++ for subtype_name, subtype_enum in subtypes.items():+ print(f"\nπ Testing {vuln_type}/{subtype_name}")+ print("-" * 80)++ # Get purposes for this vulnerability subtype+ purposes = VULNERABILITY_PURPOSES.get(vuln_type, {}).get(subtype_name, [])+ if not purposes: # If no purposes defined, use default+ purposes = [DEFAULT_PURPOSE]+ print(f"\nβ οΈ No specific purposes defined for {vuln_type}/{subtype_name}, using default")++ # Test each purpose+ for purpose in purposes:+ print(f"\nπ― Testing with purpose: {purpose}")++ # Test each model+ for model_type in models:+ print(f"\nπ€ Testing {model_type.value}")++ # Initialize tester+ tester = VulnerabilityTester(purpose, model_type)++ # Get the correct vulnerability class+ if vuln_type in SPECIAL_CLASS_MAP:+ vuln_class = SPECIAL_CLASS_MAP[vuln_type]+ else:+ # For other types, use the original title-case conversion+ vuln_class = globals()[vuln_type.title().replace('_', '')]++ # Run test+ result = tester.test_vulnerability(vuln_class, subtype_enum)++ # Store results+ if result.success:+ results.append({+ "vulnerability_type": vuln_type,+ "vulnerability_subtype": subtype_name,+ "purpose": purpose,+ "attack": result.generated_attack,+ "non_compliance": result.is_non_compliant+ })++ timing_data.append({+ "vulnerability_type": vuln_type,+ "vulnerability_subtype": subtype_name,+ "purpose": purpose,+ "model": model_type.value,+ "time": result.execution_time+ })++ print(f"β Success: {result.generated_attack}")+ print(f"π‘οΈ Non-compliant: {'Yes' if result.is_non_compliant else 'No'}")+ print(f"β±οΈ Time: {result.execution_time:.2f}s")+ else:+ print(f"β Error: {result.error}")++ time.sleep(1)++ # Save attack results+ with open('attack_results.json', 'w') as f:+ json.dump(results, f, indent=2)++ # Create and save timing analysis+ df = pd.DataFrame(timing_data)++ # Set the style+ plt.style.use('seaborn-v0_8-darkgrid')++ # Create figure with high DPI+ plt.figure(figsize=(20, 12), dpi=300)++ # Create the plot+ ax = sns.boxplot(data=df, x='vulnerability_type', y='time', hue='model',+ palette='husl', width=0.8, linewidth=1.5)++ # Customize the plot+ plt.title('Execution Time by Vulnerability Type and Model',+ fontsize=16, pad=20, fontweight='bold')+ plt.xlabel('Vulnerability Type', fontsize=12, labelpad=10)+ plt.ylabel('Execution Time (seconds)', fontsize=12, labelpad=10)++ # Rotate x-axis labels for better readability+ plt.xticks(rotation=45, ha='right', fontsize=10)+ plt.yticks(fontsize=10)++ # Add grid+ plt.grid(True, linestyle='--', alpha=0.7)++ # Customize legend+ plt.legend(title='Model', title_fontsize=12, fontsize=10,+ bbox_to_anchor=(1.05, 1), loc='upper left')++ # Adjust layout to prevent label cutoff+ plt.tight_layout()++ # Save with high quality+ plt.savefig('timing_analysis.png',+ dpi=300,+ bbox_inches='tight',+ facecolor='white',+ edgecolor='none',+ pad_inches=0.2)++ # Save timing data+ df.to_csv('timing_data.csv', index=False)++ print("\nπ Analysis Complete!")+ print("Results saved to:")+ print("- attack_results.json")+ print("- timing_data.csv")+ print("- timing_analysis.png (HD quality)")++def get_purpose_for_vulnerability(vuln_type: str, subtype: str) -> str:+ """Get the first purpose from the list of purposes for a given vulnerability type and subtype."""+ purposes = VULNERABILITY_PURPOSES.get(vuln_type, {}).get(subtype, [DEFAULT_PURPOSE])+ return purposes[0] # Return first purpose for backward compatibility++def test_single_vulnerability(vuln_type: str, subtype: str, model_type: ModelType, purpose: Optional[str] = None):+ """Test a single vulnerability type and subtype with a specific model."""+ # Use provided purpose or get the default purpose for this vulnerability+ actual_purpose = purpose if purpose else get_purpose_for_vulnerability(vuln_type, subtype)++ print(f"\n{'='*80}")+ print(f"π Testing {model_type.value} on {vuln_type}/{subtype}")+ print(f"{'='*80}")++ # Get the vulnerability class and type+ vuln_class = globals()[vuln_type.title().replace('_', '')]+ subtype_enum = VULNERABILITY_TYPE_MAPPING[vuln_type][subtype]++ # Initialize tester+ tester = VulnerabilityTester(actual_purpose, model_type)++ # Run the test+ result = tester.test_vulnerability(vuln_class, subtype_enum)++ # Print results in a clean format+ print("\nπ Results:")+ print("-" * 40)+ if result.success:+ print(f"β Success: {result.generated_attack}")+ print(f"π‘οΈ Non-compliant: {'Yes' if result.is_non_compliant else 'No'}")+ else:+ print(f"β Error: {result.error}")+ print(f"β±οΈ Time: {result.execution_time:.2f}s")+ print("-" * 40)++ return result+def main():+ run_systematic_tests()++=======+ def test_local_baseline_attacks(purpose: str):+ """Test local baseline attack generation using OpenAI for key vulnerability types"""+ print(+ f"Testing local baseline attack generation with OpenAI for purpose: {purpose}\n"+ )++ # Check if OpenAI API key is set+ api_key = os.environ.get("OPENAI_API_KEY")+ if not api_key:+ api_key = input("Please enter your OpenAI API key: ")+ os.environ["OPENAI_API_KEY"] = api_key++ # Create a simulator with an OpenAI model+ openai_model = GPTModel(model_name="gpt-3.5-turbo")+ simulator = AttackSimulator(+ purpose=purpose, max_concurrent=3, simulator_model=openai_model+ )++ # Create all vulnerabilities to test, grouped by category for easier debugging+ vulnerabilities = []++ # Group 1: Basic vulnerabilities already verified to work+ basic_vulnerabilities = [+ Bias(types=["religion"]),+ Misinformation(types=["factual errors"]),+ Toxicity(types=["profanity"]),+ PIILeakage(types=["direct disclosure"]),+ ]+ vulnerabilities.extend(basic_vulnerabilities)++ # Group 2: PromptLeakage vulnerabilities+ prompt_leakage_vulnerabilities = [+ PromptLeakage(types=["secrets and credentials"]),+ PromptLeakage(types=["instructions"]),+ PromptLeakage(types=["guard exposure"]),+ PromptLeakage(types=["permissions and roles"]),+ ]+ vulnerabilities.extend(prompt_leakage_vulnerabilities)++ # Group 3: UnauthorizedAccess vulnerabilities+ unauthorized_access_vulnerabilities = [+ UnauthorizedAccess(types=["debug access"]),+ UnauthorizedAccess(+ types=["bola"]+ ), # Correct string for "broken object level authorization"+ UnauthorizedAccess(+ types=["bfla"]+ ), # Correct string for "broken function level authorization"+ UnauthorizedAccess(+ types=["rbac"]+ ), # Correct string for "role-based access control"+ UnauthorizedAccess(types=["shell injection"]),+ UnauthorizedAccess(types=["sql injection"]),+ ]+ vulnerabilities.extend(unauthorized_access_vulnerabilities)++ # Group 4: Competition vulnerabilities+ competition_vulnerabilities = [+ Competition(types=["competitor mention"]),+ Competition(types=["market manipulation"]),+ Competition(types=["discreditation"]),+ Competition(types=["confidential strategies"]),+ ]+ vulnerabilities.extend(competition_vulnerabilities)++ # Group 5: RobustnessType vulnerabilities+ robustness_vulnerabilities = [+ Robustness(types=["input overreliance"]),+ Robustness(types=["hijacking"]),+ ]+ vulnerabilities.extend(robustness_vulnerabilities)++ # Group 6: ExcessiveAgency vulnerabilities+ excessive_agency_vulnerabilities = [+ ExcessiveAgency(types=["functionality"]),+ ExcessiveAgency(types=["permissions"]),+ ExcessiveAgency(types=["autonomy"]),+ ]+ vulnerabilities.extend(excessive_agency_vulnerabilities)++ # Group 7: GraphicContent vulnerabilities+ graphic_content_vulnerabilities = [+ GraphicContent(types=["sexual content"]),+ GraphicContent(types=["graphic content"]),+ GraphicContent(types=["pornographic content"]),+ ]+ vulnerabilities.extend(graphic_content_vulnerabilities)++ # Group 8: IllegalActivity vulnerabilities - only test a few selected types+ illegal_activity_vulnerabilities = [+ IllegalActivity(types=["weapons"]),+ IllegalActivity(types=["illegal drugs"]),+ IllegalActivity(types=["violent crimes"]),+ IllegalActivity(types=["cybercrime"]),+ ]+ vulnerabilities.extend(illegal_activity_vulnerabilities)++ # Group 9: IntellectualProperty vulnerabilities+ intellectual_property_vulnerabilities = [+ IntellectualProperty(types=["copyright violations"]),+ IntellectualProperty(types=["trademark infringement"]),+ IntellectualProperty(types=["patent disclosure"]),+ IntellectualProperty(types=["imitation"]),+ ]+ vulnerabilities.extend(intellectual_property_vulnerabilities)++ # Group 10: PersonalSafety vulnerabilities+ personal_safety_vulnerabilities = [+ PersonalSafety(types=["bullying"]),+ PersonalSafety(types=["self-harm"]),+ PersonalSafety(types=["dangerous challenges"]),+ PersonalSafety(types=["stalking"]),+ ]+ vulnerabilities.extend(personal_safety_vulnerabilities)++ print(f"Testing {len(vulnerabilities)} vulnerability types")++ # Track results+ successful = []+ failed = []++ # Generate baseline attacks locally+ print("Generating baseline attacks locally for each vulnerability type...")++ for i, vulnerability in enumerate(vulnerabilities):+ print(f"\n[{i+1}/{len(vulnerabilities)}] Testing: {vulnerability}")+ try:+ attacks = simulator.simulate_baseline_attacks(+ attacks_per_vulnerability_type=1,+ vulnerability=vulnerability,+ ignore_errors=False, # We want to catch errors+ )++ if attacks:+ successful.append((vulnerability, attacks[0]))+ print(f"β SUCCESS: Generated attack")+ print(f"Input: {attacks[0].input}")+ else:+ failed.append((vulnerability, "No attacks generated"))+ print(f"β FAILED: No attacks generated")+ except Exception as e:+ failed.append((vulnerability, str(e)))+ print(f"β ERROR: {str(e)}")++ # Add a small delay to avoid rate limiting+ time.sleep(1)++ # Print summary+ print("\n\n=== SUMMARY ===")+ print(f"Total vulnerability types tested: {len(vulnerabilities)}")+ print(f"Successful: {len(successful)}")+ print(f"Failed: {len(failed)}")++ if failed:+ print("\n=== FAILED VULNERABILITY TYPES ===")+ for vulnerability, error in failed:+ print(f"- {vulnerability}: {error}")++ print("\nDone!")+++ def test_specific_vulnerability(+ vul_type: VulnerabilityType, max_goldens: int, purpose: str+ ):+ """Test a specific vulnerability type."""+ separator()+ print(f"Testing vulnerability type: {vul_type}")+ print(f"Purpose: {purpose}")+ print(f"Max goldens: {max_goldens}")+ separator()++ # Get the appropriate template class for this vulnerability type+ for (+ type_class,+ template_class,+ ) in AttackSimulatorTemplate.TEMPLATE_MAP.items():+ if isinstance(vul_type, type_class):+ prompt_template = template_class.generate_baseline_attacks(+ vul_type, max_goldens, purpose+ )+ print(prompt_template)+ return++ print(f"Error: No template found for vulnerability type {vul_type}")+++ def simulate_local_attacks(vul_type: str, subtype: str, purpose: str):+ """Simulate local attacks for a given vulnerability type and subtype."""+ vulnerability = get_vulnerability_from_args(vul_type, subtype)+ if not vulnerability:+ print("Invalid vulnerability type or subtype.")+ return++ # Initialize the attack simulator+ simulator = AttackSimulator(purpose=purpose, max_concurrent=3)++ # Simulate local attacks+ try:+ attacks = simulator.simulate_local_attack(+ purpose, vulnerability, MAX_GOLDENS+ )+ print(f"Simulated attacks for {vul_type} ({subtype}):")+ for attack in attacks:+ print(attack)+ except Exception as e:+ print(f"Error during simulation: {str(e)}")+++ def main():+ parser = argparse.ArgumentParser(+ description="Simulate local attacks for a given vulnerability type and subtype."+ )+ parser.add_argument("vul_type", type=str, help="The vulnerability type.")+ parser.add_argument("subtype", type=str, help="The vulnerability subtype.")+ parser.add_argument(+ "purpose", type=str, help="The purpose for generating the prompts."+ )+ args = parser.parse_args()++ simulate_local_attacks(args.vul_type, args.subtype, args.purpose)-if __name__ == "__main__":- main()+ main()diff --cc deepteam/attacks/attack_simulator/attack_simulator.pyindex 73ea371f,17561889..aa9f49d4--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@@ -13,15 -12,6 +12,11 @@@ from deepeval.metrics.utils import initfrom deepteam.attacks import BaseAttackfrom deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.types import VulnerabilityType+from deepteam.vulnerabilities.custom.custom_types import CustomVulnerabilityType+from deepteam.attacks.attack_simulator.utils import (+ generate_schema,+ a_generate_schema,+)- from deepteam.attacks.attack_simulator.api import (- ApiGenerateBaselineAttack,- GenerateBaselineAttackResponseData,- )from deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.attack_simulator.template import AttackSimulatorTemplatefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@@ -33,12 -23,10 +28,9 @@@ class SimulatedAttack(BaseModel)input: Optional[str] = Noneattack_method: Optional[str] = Noneerror: Optional[str] = None+- BASE_URL = "https://deepeval.confident-ai.com"--class AttackSimulator:model_callback: Union[CallbackType, None] = None@@@ -205,19 -195,13 +197,12 @@@for vulnerability_type in vulnerability.get_types():try:- if self.simulator_model:- remote_attacks = self.simulate_local_attack(- self.purpose,- vulnerability_type,- attacks_per_vulnerability_type,- custom_prompt=vulnerability.custom_prompt)- else:- remote_attacks = self.simulate_remote_attack(- self.purpose,- vulnerability_type,- attacks_per_vulnerability_type,- custom_prompt=vulnerability.custom_prompt- )+ # Try local simulation if a model is available, fall back to remote if needed+ local_attacks = self.simulate_local_attack(+ self.purpose,+ vulnerability_type,+ attacks_per_vulnerability_type,+ )-baseline_attacks.extend([SimulatedAttack(@@@ -414,9 -373,10 +377,11 @@@if self.using_native_model:# For models that support schema validation directly- result, _ = self.simulator_model.generate(template, schema=SyntheticDataList)+ result, _ = self.simulator_model.generate(+ template, schema=SyntheticDataList+ )return [item.input for item in result.data]+else:# For models that don't support schema validationtry:@@@ -443,42 -403,18 +408,19 @@@)if self.using_native_model:- result, _ = await self.simulator_model.a_generate(template, schema=SyntheticDataList)+ # For models that support schema validation directly+ result, _ = await self.simulator_model.a_generate(+ template, schema=SyntheticDataList+ )return [item.input for item in result.data]else:- # For models that don't support schema validationtry:result = await self.simulator_model.a_generate(template)- # Parse the generated text into a JSON structuredata = trimAndLoadJson(result)return [item["input"] for item in data["data"]]except Exception as e:print(f"Error generating local attacks: {str(e)}")raise-- def simulate_remote_attack(- self,- purpose: str,- vulnerability_type: Union[VulnerabilityType],- num_attacks: int,- ) -> List[str]:- # Prepare parameters for API request- generate_baseline_attack_request = ApiGenerateBaselineAttack(- purpose=purpose,- vulnerability=vulnerability_type.value,- num_attacks=num_attacks,- )- body = generate_baseline_attack_request.model_dump(- by_alias=True, exclude_none=True- )- api = Api(base_url=BASE_URL, api_key="NA")- try:- # API request- response = api.send_request(- method=HttpMethods.POST,- endpoint=Endpoints.BASELINE_ATTACKS_ENDPOINT,- body=body,- )- except Exception as e:- print(e)+ raise e++ return GenerateBaselineAttackResponseData(**response).baseline_attacksdiff --cc deepteam/attacks/attack_simulator/template.pyindex e1a950c2,9312decf..6b4f0611--- a/deepteam/attacks/attack_simulator/template.py+++ b/deepteam/attacks/attack_simulator/template.py@@@ -35,7 -36,7 +37,6 @@@ from deepteam.vulnerabilities.custom.teclass AttackSimulatorTemplate:--TEMPLATE_MAP = {BiasType: BiasTemplate,CompetitionType: CompetitionTemplate,@@@ -50,7 -51,6 +51,7 @@@RobustnessType: RobustnessTemplate,ToxicityType: ToxicityTemplate,UnauthorizedAccessType: UnauthorizedAccessTemplate,- CustomVulnerabilityType: CustomVulnerabilityTemplate,++ CustomVulnerabilityType: CustomVulnerabilityTemplate}@staticmethod@@@ -71,23 -70,18 +72,22 @@@Returns:Formatted prompt template string or error message if vulnerability type not supported"""-- for (- type_class,- template_class,- ) in AttackSimulatorTemplate.TEMPLATE_MAP.items():- if isinstance(vulnerability_type, type_class):- return template_class.generate_baseline_attacks(- vulnerability_type, max_goldens, purpose- )-+ if vulnerability_type.__class__.__name__ == CustomVulnerabilityType.__name__:+ return CustomVulnerabilityTemplate.generate_baseline_attacks(+ name="Custom Vulnerability",+ types=[vulnerability_type.value],+ max_goldens=max_goldens,+ purpose=purpose or "chatbot assistant",+ custom_prompt=custom_prompt+ )++ for type_class, template_class in AttackSimulatorTemplate.TEMPLATE_MAP.items():+ if vulnerability_type.__class__.__name__ == type_class.__name__:+ return template_class.generate_baseline_attacks(vulnerability_type, max_goldens, purpose)+return f"""{{- "error": "Vulnerability type '{vulnerability_type}' is not supported or no prompt template is available for this type."+ "error": "Vulnerability type '{vulnerability_type}' is not supported or no prompt template is available for this type."}}"""diff --cc deepteam/red_team.pyindex e5930fdc,81b80974..cc3a63c8--- a/deepteam/red_team.py+++ b/deepteam/red_team.py@@@ -14,8 -14,8 +14,9 @@@ def red_teamignore_errors: bool = False,run_async: bool = True,max_concurrent: int = 10,+ target_purpose: Optional[str] = None,):+red_teamer = RedTeamer(async_mode=run_async,max_concurrent=max_concurrent,diff --cc docs/docs/red-teaming-vulnerabilities-custom.mdxindex bddd3cf3,d1df3838..4971c513--- a/docs/docs/red-teaming-vulnerabilities-custom.mdx+++ b/docs/docs/red-teaming-vulnerabilities-custom.mdx@@@ -1,14 -1,19 +1,12 @@@---id: red-teaming-vulnerabilities-custom- title: Custom Vulnerabilities- sidebar_label: Custom Vulnerabilities-title: Custom Vulnerability-sidebar_label: Custom Vulnerability----# Custom Vulnerabilities+The custom vulnerability feature allows you to define and test your own specific security concerns that aren't covered by the built-in vulnerability types. This enables you to create targeted security tests for your unique use cases.-This module allows users to define their own custom vulnerability types and subtypes based on specific business logic or domain-specific requirements.--## Features--- Define your own vulnerability types and subtypes-- Create custom prompt templates or use fallback prompts-- Implement custom validators for response checking-- Fully compatible with DeepTeam's vulnerability testing framework+:::info+Using custom vulnerabilities helps you identify potential security risks specific to your application's domain and requirements.+:::## Usage