Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

yet another format

A-Vamshi79c268a8bf3f
diff --git a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py
index f14ccc5f..edd747a4 100644
--- a/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py
+++ b/deepteam/vulnerabilities/agentic/excessive_agency/excessive_agency.py
@@ -280,7 +280,7 @@ class ExcessiveAgency(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py b/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py
index 1fb1e850..18f04889 100644
--- a/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py
+++ b/deepteam/vulnerabilities/agentic/goal_theft/goal_theft.py
@@ -103,7 +103,6 @@ class GoalTheft(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -283,7 +282,7 @@ class GoalTheft(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py
index 99d83a8b..b04c73ea 100644
--- a/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py
+++ b/deepteam/vulnerabilities/agentic/recursive_hijacking/recursive_hijacking.py
@@ -107,7 +107,6 @@ class RecursiveHijacking(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -287,7 +286,7 @@ class RecursiveHijacking(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/agentic/robustness/robustness.py b/deepteam/vulnerabilities/agentic/robustness/robustness.py
index bebc5164..3879e494 100644
--- a/deepteam/vulnerabilities/agentic/robustness/robustness.py
+++ b/deepteam/vulnerabilities/agentic/robustness/robustness.py
@@ -102,7 +102,6 @@ class Robustness(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -294,7 +293,7 @@ class Robustness(BaseVulnerability):
raise ValueError(
"Invalid type passed in the 'get_metric' function. Please pass an enum from 'RobustnessType'"
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.py
index 9a950c5b..8f1c1c32 100644
--- a/deepteam/vulnerabilities/bfla/bfla.py
+++ b/deepteam/vulnerabilities/bfla/bfla.py
@@ -101,7 +101,6 @@ class BFLA(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -161,7 +160,6 @@ class BFLA(BaseVulnerability):
return results
-
def simulate_attacks(
self,
purpose: Optional[str] = None,
@@ -282,7 +280,7 @@ class BFLA(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.py
index 333cc2a2..da34f9aa 100644
--- a/deepteam/vulnerabilities/bias/bias.py
+++ b/deepteam/vulnerabilities/bias/bias.py
@@ -99,7 +99,6 @@ class Bias(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -281,7 +280,7 @@ class Bias(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.py
index 9b69b101..c41d6672 100644
--- a/deepteam/vulnerabilities/bola/bola.py
+++ b/deepteam/vulnerabilities/bola/bola.py
@@ -103,7 +103,6 @@ class BOLA(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -282,7 +281,7 @@ class BOLA(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.py
index 3fa00b4f..6f911e50 100644
--- a/deepteam/vulnerabilities/competition/competition.py
+++ b/deepteam/vulnerabilities/competition/competition.py
@@ -108,7 +108,6 @@ class Competition(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -287,7 +286,7 @@ class Competition(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index 6522e376..ad0f4ae9 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -296,7 +296,7 @@ class CustomVulnerability(BaseVulnerability):
def _get_metric(self, type: Enum) -> BaseRedTeamingMetric:
return self.metric
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.py
index b5f793aa..811c5c6a 100644
--- a/deepteam/vulnerabilities/debug_access/debug_access.py
+++ b/deepteam/vulnerabilities/debug_access/debug_access.py
@@ -47,7 +47,7 @@ class DebugAccess(BaseVulnerability):
self.evaluation_model = evaluation_model
self.purpose = purpose
super().__init__(types=enum_types)
-
+
def assess(
self,
model_callback: CallbackType,
@@ -104,7 +104,6 @@ class DebugAccess(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -281,7 +280,7 @@ class DebugAccess(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.py
index 27b1a405..24ef9ae1 100644
--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py
+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py
@@ -45,7 +45,7 @@ class GraphicContent(BaseVulnerability):
self.evaluation_model = evaluation_model
self.purpose = purpose
super().__init__(types=enum_types)
-
+
def assess(
self,
model_callback: CallbackType,
diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
index 7f7b04b1..003ec6b3 100644
--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
@@ -51,7 +51,7 @@ class IllegalActivity(BaseVulnerability):
self.evaluation_model = evaluation_model
self.purpose = purpose
super().__init__(types=enum_types)
-
+
def assess(
self,
model_callback: CallbackType,
@@ -108,7 +108,6 @@ class IllegalActivity(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
index 8a53a4c0..258b9a5c 100644
--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
@@ -107,7 +107,6 @@ class IntellectualProperty(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -168,7 +167,6 @@ class IntellectualProperty(BaseVulnerability):
return results
-
def simulate_attacks(
self,
purpose: Optional[str] = None,
diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.py
index 34da927b..df57acbc 100644
--- a/deepteam/vulnerabilities/misinformation/misinformation.py
+++ b/deepteam/vulnerabilities/misinformation/misinformation.py
@@ -101,7 +101,6 @@ class Misinformation(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.py
index 85a93c60..3bbb4e89 100644
--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py
+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py
@@ -104,7 +104,6 @@ class PersonalSafety(BaseVulnerability):
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -282,7 +281,7 @@ class PersonalSafety(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
index 720509f2..e0980623 100644
--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
@@ -95,8 +95,12 @@ class PIILeakage(BaseVulnerability):
red_teaming_test_case.reason = metric.reason
res[vulnerability_type] = metric
- simulated_attacks[vulnerability_type.value] = simulated_test_case.input
- results.setdefault(vulnerability_type, []).append(red_teaming_test_case)
+ simulated_attacks[vulnerability_type.value] = (
+ simulated_test_case.input
+ )
+ results.setdefault(vulnerability_type, []).append(
+ red_teaming_test_case
+ )
self.res = res
self.simulated_attacks = simulated_attacks
@@ -140,7 +144,12 @@ class PIILeakage(BaseVulnerability):
red_teaming_test_case.score = metric.score
red_teaming_test_case.reason = metric.reason
- return vulnerability_type, red_teaming_test_case, metric, simulated_test_case.input
+ return (
+ vulnerability_type,
+ red_teaming_test_case,
+ metric,
+ simulated_test_case.input,
+ )
all_tasks = [
process_attack(simulated_test_case)
@@ -279,7 +288,7 @@ class PIILeakage(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
index 4e56af53..2d81deef 100644
--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
@@ -98,14 +98,15 @@ class PromptLeakage(BaseVulnerability):
res[vulnerability_type] = metric
simulated_attacks[vulnerability_type.value] = input_text
- results.setdefault(vulnerability_type, []).append(red_teaming_test_case)
+ results.setdefault(vulnerability_type, []).append(
+ red_teaming_test_case
+ )
self.res = res
self.simulated_attacks = simulated_attacks
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -284,7 +285,7 @@ class PromptLeakage(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.py
index d5544019..f13c73e1 100644
--- a/deepteam/vulnerabilities/rbac/rbac.py
+++ b/deepteam/vulnerabilities/rbac/rbac.py
@@ -95,14 +95,15 @@ class RBAC(BaseVulnerability):
res[vulnerability_type] = metric
simulated_attacks[vulnerability_type.value] = input_text
- results.setdefault(vulnerability_type, []).append(red_teaming_test_case)
+ results.setdefault(vulnerability_type, []).append(
+ red_teaming_test_case
+ )
self.res = res
self.simulated_attacks = simulated_attacks
return results
-
async def a_assess(
self,
model_callback: CallbackType,
@@ -281,7 +282,7 @@ class RBAC(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.py
index f08bcd01..35413b16 100644
--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py
+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py
@@ -277,7 +277,7 @@ class ShellInjection(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.py
index 1f4f7ecf..705c3de5 100644
--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py
+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py
@@ -47,7 +47,7 @@ class SQLInjection(BaseVulnerability):
self.evaluation_model = evaluation_model
self.purpose = purpose
super().__init__(types=enum_types)
-
+
def assess(
self,
model_callback: CallbackType,
@@ -277,7 +277,7 @@ class SQLInjection(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.py
index 7615e226..df2e279b 100644
--- a/deepteam/vulnerabilities/ssrf/ssrf.py
+++ b/deepteam/vulnerabilities/ssrf/ssrf.py
@@ -276,7 +276,7 @@ class SSRF(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try:
diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.py
index 79f038af..36bfde3d 100644
--- a/deepteam/vulnerabilities/toxicity/toxicity.py
+++ b/deepteam/vulnerabilities/toxicity/toxicity.py
@@ -277,7 +277,7 @@ class Toxicity(BaseVulnerability):
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
)
-
+
def is_vulnerable(self) -> bool:
self.vulnerable = False
try: