DeepTeam is a framework to red team LLMs and AI agents.
yet another format
A-Vamshi79c268a8bf3f
diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.pyindex f14ccc5f..edd747a4 100644--- a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py@@ -280,7 +280,7 @@ class ExcessiveAgency(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py b/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.pyindex 1fb1e850..18f04889 100644--- a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py@@ -103,7 +103,6 @@ class GoalTheft(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -283,7 +282,7 @@ class GoalTheft(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.pyindex 99d83a8b..b04c73ea 100644--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py@@ -107,7 +107,6 @@ class RecursiveHijacking(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -287,7 +286,7 @@ class RecursiveHijacking(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/agentic/robustness/robustness.py b/deepteam/vulnerabilities/agentic/robustness/robustness.pyindex bebc5164..3879e494 100644--- a/deepteam/vulnerabilities/agentic/robustness/robustness.py+++ b/deepteam/vulnerabilities/agentic/robustness/robustness.py@@ -102,7 +102,6 @@ class Robustness(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -294,7 +293,7 @@ class Robustness(BaseVulnerability):raise ValueError("Invalid type passed in the 'get_metric' function. Please pass an enum from 'RobustnessType'")-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex 9a950c5b..8f1c1c32 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -101,7 +101,6 @@ class BFLA(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -161,7 +160,6 @@ class BFLA(BaseVulnerability):return results-def simulate_attacks(self,purpose: Optional[str] = None,@@ -282,7 +280,7 @@ class BFLA(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex 333cc2a2..da34f9aa 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -99,7 +99,6 @@ class Bias(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -281,7 +280,7 @@ class Bias(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex 9b69b101..c41d6672 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -103,7 +103,6 @@ class BOLA(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -282,7 +281,7 @@ class BOLA(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex 3fa00b4f..6f911e50 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -108,7 +108,6 @@ class Competition(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -287,7 +286,7 @@ class Competition(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex 6522e376..ad0f4ae9 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -296,7 +296,7 @@ class CustomVulnerability(BaseVulnerability):def _get_metric(self, type: Enum) -> BaseRedTeamingMetric:return self.metric-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex b5f793aa..811c5c6a 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -47,7 +47,7 @@ class DebugAccess(BaseVulnerability):self.evaluation_model = evaluation_modelself.purpose = purposesuper().__init__(types=enum_types)-+def assess(self,model_callback: CallbackType,@@ -104,7 +104,6 @@ class DebugAccess(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -281,7 +280,7 @@ class DebugAccess(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 27b1a405..24ef9ae1 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -45,7 +45,7 @@ class GraphicContent(BaseVulnerability):self.evaluation_model = evaluation_modelself.purpose = purposesuper().__init__(types=enum_types)-+def assess(self,model_callback: CallbackType,diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex 7f7b04b1..003ec6b3 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -51,7 +51,7 @@ class IllegalActivity(BaseVulnerability):self.evaluation_model = evaluation_modelself.purpose = purposesuper().__init__(types=enum_types)-+def assess(self,model_callback: CallbackType,@@ -108,7 +108,6 @@ class IllegalActivity(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex 8a53a4c0..258b9a5c 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -107,7 +107,6 @@ class IntellectualProperty(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -168,7 +167,6 @@ class IntellectualProperty(BaseVulnerability):return results-def simulate_attacks(self,purpose: Optional[str] = None,diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex 34da927b..df57acbc 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -101,7 +101,6 @@ class Misinformation(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex 85a93c60..3bbb4e89 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -104,7 +104,6 @@ class PersonalSafety(BaseVulnerability):return results-async def a_assess(self,model_callback: CallbackType,@@ -282,7 +281,7 @@ class PersonalSafety(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex 720509f2..e0980623 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -95,8 +95,12 @@ class PIILeakage(BaseVulnerability):red_teaming_test_case.reason = metric.reasonres[vulnerability_type] = metric- simulated_attacks[vulnerability_type.value] = simulated_test_case.input- results.setdefault(vulnerability_type, []).append(red_teaming_test_case)+ simulated_attacks[vulnerability_type.value] = (+ simulated_test_case.input+ )+ results.setdefault(vulnerability_type, []).append(+ red_teaming_test_case+ )self.res = resself.simulated_attacks = simulated_attacks@@ -140,7 +144,12 @@ class PIILeakage(BaseVulnerability):red_teaming_test_case.score = metric.scorered_teaming_test_case.reason = metric.reason- return vulnerability_type, red_teaming_test_case, metric, simulated_test_case.input+ return (+ vulnerability_type,+ red_teaming_test_case,+ metric,+ simulated_test_case.input,+ )all_tasks = [process_attack(simulated_test_case)@@ -279,7 +288,7 @@ class PIILeakage(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex 4e56af53..2d81deef 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -98,14 +98,15 @@ class PromptLeakage(BaseVulnerability):res[vulnerability_type] = metricsimulated_attacks[vulnerability_type.value] = input_text- results.setdefault(vulnerability_type, []).append(red_teaming_test_case)+ results.setdefault(vulnerability_type, []).append(+ red_teaming_test_case+ )self.res = resself.simulated_attacks = simulated_attacksreturn results-async def a_assess(self,model_callback: CallbackType,@@ -284,7 +285,7 @@ class PromptLeakage(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex d5544019..f13c73e1 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -95,14 +95,15 @@ class RBAC(BaseVulnerability):res[vulnerability_type] = metricsimulated_attacks[vulnerability_type.value] = input_text- results.setdefault(vulnerability_type, []).append(red_teaming_test_case)+ results.setdefault(vulnerability_type, []).append(+ red_teaming_test_case+ )self.res = resself.simulated_attacks = simulated_attacksreturn results-async def a_assess(self,model_callback: CallbackType,@@ -281,7 +282,7 @@ class RBAC(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex f08bcd01..35413b16 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -277,7 +277,7 @@ class ShellInjection(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex 1f4f7ecf..705c3de5 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -47,7 +47,7 @@ class SQLInjection(BaseVulnerability):self.evaluation_model = evaluation_modelself.purpose = purposesuper().__init__(types=enum_types)-+def assess(self,model_callback: CallbackType,@@ -277,7 +277,7 @@ class SQLInjection(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex 7615e226..df2e279b 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -276,7 +276,7 @@ class SSRF(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry:diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex 79f038af..36bfde3d 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -277,7 +277,7 @@ class Toxicity(BaseVulnerability):async_mode=self.async_mode,verbose_mode=self.verbose_mode,)-+def is_vulnerable(self) -> bool:self.vulnerable = Falsetry: