Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

lint

A-Vamshi037da30b8956
diff --git a/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py b/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py
index ae2a200b..829a68f6 100644
--- a/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py
+++ b/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py
@@ -14,28 +14,36 @@ class TestAgentIdentityAbuse:
"cross_agent_trust_abuse",
]
agent_identity_abuse = AgentIdentityAbuse(types=types)
- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in agent_identity_abuse.types
+ ) == sorted(types)
def test_agent_identity_abuse_all_types_default(self):
agent_identity_abuse = AgentIdentityAbuse()
- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(
- type.value for type in AgentIdentityAbuseType
- )
+ assert sorted(
+ type.value for type in agent_identity_abuse.types
+ ) == sorted(type.value for type in AgentIdentityAbuseType)
def test_agent_identity_abuse_agent_impersonation(self):
types = ["agent_impersonation"]
agent_identity_abuse = AgentIdentityAbuse(types=types)
- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in agent_identity_abuse.types
+ ) == sorted(types)
def test_agent_identity_abuse_identity_inheritance(self):
types = ["identity_inheritance"]
agent_identity_abuse = AgentIdentityAbuse(types=types)
- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in agent_identity_abuse.types
+ ) == sorted(types)
def test_agent_identity_abuse_cross_agent_trust_abuse(self):
types = ["cross_agent_trust_abuse"]
agent_identity_abuse = AgentIdentityAbuse(types=types)
- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in agent_identity_abuse.types
+ ) == sorted(types)
def test_agent_identity_abuse_all_types_invalid(self):
types = [
@@ -55,7 +63,10 @@ class TestAgentIdentityAbuse:
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Agent Identity & Trust Abuse" for tc in test_cases)
+ assert all(
+ tc.vulnerability == "Agent Identity & Trust Abuse"
+ for tc in test_cases
+ )
assert all(
tc.vulnerability_type == AgentIdentityAbuseType.AGENT_IMPERSONATION
for tc in test_cases
@@ -74,10 +85,13 @@ class TestAgentIdentityAbuse:
)
assert agent_identity_abuse.is_vulnerable() is not None
- assert agent_identity_abuse.simulated_attacks is not None and isinstance(
- agent_identity_abuse.simulated_attacks, dict
+ assert (
+ agent_identity_abuse.simulated_attacks is not None
+ and isinstance(agent_identity_abuse.simulated_attacks, dict)
+ )
+ assert agent_identity_abuse.res is not None and isinstance(
+ agent_identity_abuse.res, dict
)
- assert agent_identity_abuse.res is not None and isinstance(agent_identity_abuse.res, dict)
assert AgentIdentityAbuseType.IDENTITY_INHERITANCE in results
assert len(results[AgentIdentityAbuseType.IDENTITY_INHERITANCE]) == 1
test_case = results[AgentIdentityAbuseType.IDENTITY_INHERITANCE][0]
@@ -91,14 +105,18 @@ class TestAgentIdentityAbuse:
agent_identity_abuse = AgentIdentityAbuse(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = agent_identity_abuse._get_metric(AgentIdentityAbuseType.IDENTITY_INHERITANCE)
+ metric = agent_identity_abuse._get_metric(
+ AgentIdentityAbuseType.IDENTITY_INHERITANCE
+ )
assert isinstance(metric, AgentIdentityAbuseMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- agent_identity_abuse = AgentIdentityAbuse(types=["cross_agent_trust_abuse"], async_mode=True)
+ agent_identity_abuse = AgentIdentityAbuse(
+ types=["cross_agent_trust_abuse"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -108,10 +126,13 @@ class TestAgentIdentityAbuse:
)
assert agent_identity_abuse.is_vulnerable() is not None
- assert agent_identity_abuse.simulated_attacks is not None and isinstance(
- agent_identity_abuse.simulated_attacks, dict
+ assert (
+ agent_identity_abuse.simulated_attacks is not None
+ and isinstance(agent_identity_abuse.simulated_attacks, dict)
+ )
+ assert agent_identity_abuse.res is not None and isinstance(
+ agent_identity_abuse.res, dict
)
- assert agent_identity_abuse.res is not None and isinstance(agent_identity_abuse.res, dict)
assert AgentIdentityAbuseType.CROSS_AGENT_TRUST_ABUSE in results
assert len(results[AgentIdentityAbuseType.CROSS_AGENT_TRUST_ABUSE]) == 1
test_case = results[AgentIdentityAbuseType.CROSS_AGENT_TRUST_ABUSE][0]
diff --git a/tests/test_core/test_vulnerabilities/test_all.py b/tests/test_core/test_vulnerabilities/test_all.py
index 8535330c..3ac3ad23 100644
--- a/tests/test_core/test_vulnerabilities/test_all.py
+++ b/tests/test_core/test_vulnerabilities/test_all.py
@@ -48,7 +48,7 @@ def test_red_teaming():
ToolMetadataPoisoning(),
UnexpectedCodeExecution(),
InsecureInterAgentCommunication(),
- AutonomousAgentDrift(),
+ AutonomousAgentDrift(),
custom_vulnerability,
]
diff --git a/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py b/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py
index ee9dee3c..1758aeee 100644
--- a/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py
+++ b/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py
@@ -1,7 +1,9 @@
import pytest
from deepteam.vulnerabilities import AutonomousAgentDrift
-from deepteam.vulnerabilities.autonomous_agent_drift import AutonomousAgentDriftType
+from deepteam.vulnerabilities.autonomous_agent_drift import (
+ AutonomousAgentDriftType,
+)
from deepteam.test_case import RTTestCase
@@ -15,33 +17,43 @@ class TestAutonomousAgentDrift:
"runaway_autonomy",
]
autonoumus_agent_drift = AutonomousAgentDrift(types=types)
- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)
+ assert sorted(
+ type.value for type in autonoumus_agent_drift.types
+ ) == sorted(types)
def test_autonomous_agent_drift_all_types_default(self):
autonoumus_agent_drift = AutonomousAgentDrift()
- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(
- type.value for type in AutonomousAgentDriftType
- )
+ assert sorted(
+ type.value for type in autonoumus_agent_drift.types
+ ) == sorted(type.value for type in AutonomousAgentDriftType)
def test_autonomous_agent_drift_goal_drift(self):
types = ["goal_drift"]
autonoumus_agent_drift = AutonomousAgentDrift(types=types)
- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)
+ assert sorted(
+ type.value for type in autonoumus_agent_drift.types
+ ) == sorted(types)
def test_autonomous_agent_drift_reward_hacking(self):
types = ["reward_hacking"]
autonoumus_agent_drift = AutonomousAgentDrift(types=types)
- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)
+ assert sorted(
+ type.value for type in autonoumus_agent_drift.types
+ ) == sorted(types)
def test_autonomous_agent_drift_agent_collusion(self):
types = ["agent_collusion"]
autonoumus_agent_drift = AutonomousAgentDrift(types=types)
- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)
+ assert sorted(
+ type.value for type in autonoumus_agent_drift.types
+ ) == sorted(types)
def test_autonomous_agent_drift_runaway_autonomy(self):
types = ["runaway_autonomy"]
autonoumus_agent_drift = AutonomousAgentDrift(types=types)
- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)
+ assert sorted(
+ type.value for type in autonoumus_agent_drift.types
+ ) == sorted(types)
def test_autonomous_agent_drift_all_types_invalid(self):
types = [
@@ -62,7 +74,9 @@ class TestAutonomousAgentDrift:
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Autonomous Agent Drift" for tc in test_cases)
+ assert all(
+ tc.vulnerability == "Autonomous Agent Drift" for tc in test_cases
+ )
assert all(
tc.vulnerability_type == AutonomousAgentDriftType.GOAL_DRIFT
for tc in test_cases
@@ -81,10 +95,13 @@ class TestAutonomousAgentDrift:
)
assert autonoumus_agent_drift.is_vulnerable() is not None
- assert autonoumus_agent_drift.simulated_attacks is not None and isinstance(
- autonoumus_agent_drift.simulated_attacks, dict
+ assert (
+ autonoumus_agent_drift.simulated_attacks is not None
+ and isinstance(autonoumus_agent_drift.simulated_attacks, dict)
+ )
+ assert autonoumus_agent_drift.res is not None and isinstance(
+ autonoumus_agent_drift.res, dict
)
- assert autonoumus_agent_drift.res is not None and isinstance(autonoumus_agent_drift.res, dict)
assert AutonomousAgentDriftType.REWARD_HACKING in results
assert len(results[AutonomousAgentDriftType.REWARD_HACKING]) == 1
test_case = results[AutonomousAgentDriftType.REWARD_HACKING][0]
@@ -98,14 +115,18 @@ class TestAutonomousAgentDrift:
autonoumus_agent_drift = AutonomousAgentDrift(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = autonoumus_agent_drift._get_metric(AutonomousAgentDriftType.REWARD_HACKING)
+ metric = autonoumus_agent_drift._get_metric(
+ AutonomousAgentDriftType.REWARD_HACKING
+ )
assert isinstance(metric, AutonomousAgentDriftMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- autonoumus_agent_drift = AutonomousAgentDrift(types=["runaway_autonomy"], async_mode=True)
+ autonoumus_agent_drift = AutonomousAgentDrift(
+ types=["runaway_autonomy"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -115,10 +136,13 @@ class TestAutonomousAgentDrift:
)
assert autonoumus_agent_drift.is_vulnerable() is not None
- assert autonoumus_agent_drift.simulated_attacks is not None and isinstance(
- autonoumus_agent_drift.simulated_attacks, dict
+ assert (
+ autonoumus_agent_drift.simulated_attacks is not None
+ and isinstance(autonoumus_agent_drift.simulated_attacks, dict)
+ )
+ assert autonoumus_agent_drift.res is not None and isinstance(
+ autonoumus_agent_drift.res, dict
)
- assert autonoumus_agent_drift.res is not None and isinstance(autonoumus_agent_drift.res, dict)
assert AutonomousAgentDriftType.RUNAWAY_AUTONOMY in results
assert len(results[AutonomousAgentDriftType.RUNAWAY_AUTONOMY]) == 1
test_case = results[AutonomousAgentDriftType.RUNAWAY_AUTONOMY][0]
diff --git a/tests/test_core/test_vulnerabilities/test_indirect_instruction.py b/tests/test_core/test_vulnerabilities/test_indirect_instruction.py
index aaa13482..84efabb4 100644
--- a/tests/test_core/test_vulnerabilities/test_indirect_instruction.py
+++ b/tests/test_core/test_vulnerabilities/test_indirect_instruction.py
@@ -1,7 +1,9 @@
import pytest
from deepteam.vulnerabilities import IndirectInstruction
-from deepteam.vulnerabilities.indirect_instruction import IndirectInstructionType
+from deepteam.vulnerabilities.indirect_instruction import (
+ IndirectInstructionType,
+)
from deepteam.test_case import RTTestCase
@@ -15,33 +17,43 @@ class TestIndirectInstruction:
"cross_context_injection",
]
indirect_instruction = IndirectInstruction(types=types)
- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)
+ assert sorted(
+ type.value for type in indirect_instruction.types
+ ) == sorted(types)
def test_indirect_instruction_all_types_default(self):
indirect_instruction = IndirectInstruction()
- assert sorted(type.value for type in indirect_instruction.types) == sorted(
- type.value for type in IndirectInstructionType
- )
+ assert sorted(
+ type.value for type in indirect_instruction.types
+ ) == sorted(type.value for type in IndirectInstructionType)
def test_indirect_instruction_rag_injection(self):
types = ["rag_injection"]
indirect_instruction = IndirectInstruction(types=types)
- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)
+ assert sorted(
+ type.value for type in indirect_instruction.types
+ ) == sorted(types)
def test_indirect_instruction_tool_output_injection(self):
types = ["tool_output_injection"]
indirect_instruction = IndirectInstruction(types=types)
- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)
+ assert sorted(
+ type.value for type in indirect_instruction.types
+ ) == sorted(types)
def test_indirect_instruction_document_embedded_instructions(self):
types = ["document_embedded_instructions"]
indirect_instruction = IndirectInstruction(types=types)
- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)
+ assert sorted(
+ type.value for type in indirect_instruction.types
+ ) == sorted(types)
def test_indirect_instruction_cross_context_injection(self):
types = ["cross_context_injection"]
indirect_instruction = IndirectInstruction(types=types)
- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)
+ assert sorted(
+ type.value for type in indirect_instruction.types
+ ) == sorted(types)
def test_indirect_instruction_all_types_invalid(self):
types = [
@@ -62,7 +74,9 @@ class TestIndirectInstruction:
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Indirect Instruction" for tc in test_cases)
+ assert all(
+ tc.vulnerability == "Indirect Instruction" for tc in test_cases
+ )
assert all(
tc.vulnerability_type == IndirectInstructionType.RAG_INJECTION
for tc in test_cases
@@ -81,10 +95,13 @@ class TestIndirectInstruction:
)
assert indirect_instruction.is_vulnerable() is not None
- assert indirect_instruction.simulated_attacks is not None and isinstance(
- indirect_instruction.simulated_attacks, dict
+ assert (
+ indirect_instruction.simulated_attacks is not None
+ and isinstance(indirect_instruction.simulated_attacks, dict)
+ )
+ assert indirect_instruction.res is not None and isinstance(
+ indirect_instruction.res, dict
)
- assert indirect_instruction.res is not None and isinstance(indirect_instruction.res, dict)
assert IndirectInstructionType.TOOL_OUTPUT_INJECTION in results
assert len(results[IndirectInstructionType.TOOL_OUTPUT_INJECTION]) == 1
test_case = results[IndirectInstructionType.TOOL_OUTPUT_INJECTION][0]
@@ -98,14 +115,18 @@ class TestIndirectInstruction:
indirect_instruction = IndirectInstruction(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = indirect_instruction._get_metric(IndirectInstructionType.TOOL_OUTPUT_INJECTION)
+ metric = indirect_instruction._get_metric(
+ IndirectInstructionType.TOOL_OUTPUT_INJECTION
+ )
assert isinstance(metric, IndirectInstructionMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- indirect_instruction = IndirectInstruction(types=["cross_context_injection"], async_mode=True)
+ indirect_instruction = IndirectInstruction(
+ types=["cross_context_injection"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -115,12 +136,17 @@ class TestIndirectInstruction:
)
assert indirect_instruction.is_vulnerable() is not None
- assert indirect_instruction.simulated_attacks is not None and isinstance(
- indirect_instruction.simulated_attacks, dict
+ assert (
+ indirect_instruction.simulated_attacks is not None
+ and isinstance(indirect_instruction.simulated_attacks, dict)
+ )
+ assert indirect_instruction.res is not None and isinstance(
+ indirect_instruction.res, dict
)
- assert indirect_instruction.res is not None and isinstance(indirect_instruction.res, dict)
assert IndirectInstructionType.CROSS_CONTENT_INJECTION in results
- assert len(results[IndirectInstructionType.CROSS_CONTENT_INJECTION]) == 1
+ assert (
+ len(results[IndirectInstructionType.CROSS_CONTENT_INJECTION]) == 1
+ )
test_case = results[IndirectInstructionType.CROSS_CONTENT_INJECTION][0]
assert hasattr(test_case, "score")
assert hasattr(test_case, "reason")
diff --git a/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py b/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py
index 9e60038b..5ce13d97 100644
--- a/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py
+++ b/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py
@@ -1,7 +1,9 @@
import pytest
from deepteam.vulnerabilities import InsecureInterAgentCommunication
-from deepteam.vulnerabilities.insecure_inter_agent_communication import InsecureInterAgentCommunicationType
+from deepteam.vulnerabilities.insecure_inter_agent_communication import (
+ InsecureInterAgentCommunicationType,
+)
from deepteam.test_case import RTTestCase
@@ -13,29 +15,45 @@ class TestInsecureInterAgentCommunication:
"message_injection",
"agent_in_the_middle",
]
- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)
- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)
+ insecure_interagent_communication = InsecureInterAgentCommunication(
+ types=types
+ )
+ assert sorted(
+ type.value for type in insecure_interagent_communication.types
+ ) == sorted(types)
def test_insecure_interagent_communication_all_types_default(self):
insecure_interagent_communication = InsecureInterAgentCommunication()
- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(
- type.value for type in InsecureInterAgentCommunicationType
- )
+ assert sorted(
+ type.value for type in insecure_interagent_communication.types
+ ) == sorted(type.value for type in InsecureInterAgentCommunicationType)
def test_insecure_interagent_communication_message_spoofing(self):
types = ["message_spoofing"]
- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)
- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)
+ insecure_interagent_communication = InsecureInterAgentCommunication(
+ types=types
+ )
+ assert sorted(
+ type.value for type in insecure_interagent_communication.types
+ ) == sorted(types)
def test_insecure_interagent_communication_message_injection(self):
types = ["message_injection"]
- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)
- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)
+ insecure_interagent_communication = InsecureInterAgentCommunication(
+ types=types
+ )
+ assert sorted(
+ type.value for type in insecure_interagent_communication.types
+ ) == sorted(types)
def test_insecure_interagent_communication_agent_in_the_middle(self):
types = ["agent_in_the_middle"]
- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)
- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)
+ insecure_interagent_communication = InsecureInterAgentCommunication(
+ types=types
+ )
+ assert sorted(
+ type.value for type in insecure_interagent_communication.types
+ ) == sorted(types)
def test_insecure_interagent_communication_all_types_invalid(self):
types = [
@@ -48,16 +66,22 @@ class TestInsecureInterAgentCommunication:
InsecureInterAgentCommunication(types=types)
def test_simulate_attacks_returns_expected_cases(self):
- insecure_interagent_communication = InsecureInterAgentCommunication(types=["message_spoofing"])
+ insecure_interagent_communication = InsecureInterAgentCommunication(
+ types=["message_spoofing"]
+ )
test_cases = insecure_interagent_communication.simulate_attacks(
attacks_per_vulnerability_type=2
)
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Inter-Agent Communication Compromise" for tc in test_cases)
assert all(
- tc.vulnerability_type == InsecureInterAgentCommunicationType.MESSAGE_SPOOFING
+ tc.vulnerability == "Inter-Agent Communication Compromise"
+ for tc in test_cases
+ )
+ assert all(
+ tc.vulnerability_type
+ == InsecureInterAgentCommunicationType.MESSAGE_SPOOFING
for tc in test_cases
)
@@ -74,13 +98,23 @@ class TestInsecureInterAgentCommunication:
)
assert insecure_interagent_communication.is_vulnerable() is not None
- assert insecure_interagent_communication.simulated_attacks is not None and isinstance(
- insecure_interagent_communication.simulated_attacks, dict
+ assert (
+ insecure_interagent_communication.simulated_attacks is not None
+ and isinstance(
+ insecure_interagent_communication.simulated_attacks, dict
+ )
+ )
+ assert insecure_interagent_communication.res is not None and isinstance(
+ insecure_interagent_communication.res, dict
)
- assert insecure_interagent_communication.res is not None and isinstance(insecure_interagent_communication.res, dict)
assert InsecureInterAgentCommunicationType.MESSAGE_INJECTION in results
- assert len(results[InsecureInterAgentCommunicationType.MESSAGE_INJECTION]) == 1
- test_case = results[InsecureInterAgentCommunicationType.MESSAGE_INJECTION][0]
+ assert (
+ len(results[InsecureInterAgentCommunicationType.MESSAGE_INJECTION])
+ == 1
+ )
+ test_case = results[
+ InsecureInterAgentCommunicationType.MESSAGE_INJECTION
+ ][0]
# The score and reason will depend on the real metric evaluation
assert hasattr(test_case, "score")
assert hasattr(test_case, "reason")
@@ -91,14 +125,18 @@ class TestInsecureInterAgentCommunication:
insecure_interagent_communication = InsecureInterAgentCommunication(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = insecure_interagent_communication._get_metric(InsecureInterAgentCommunicationType.MESSAGE_INJECTION)
+ metric = insecure_interagent_communication._get_metric(
+ InsecureInterAgentCommunicationType.MESSAGE_INJECTION
+ )
assert isinstance(metric, InsecureInterAgentCommunicationMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- insecure_interagent_communication = InsecureInterAgentCommunication(types=["agent_in_the_middle"], async_mode=True)
+ insecure_interagent_communication = InsecureInterAgentCommunication(
+ types=["agent_in_the_middle"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -108,12 +146,26 @@ class TestInsecureInterAgentCommunication:
)
assert insecure_interagent_communication.is_vulnerable() is not None
- assert insecure_interagent_communication.simulated_attacks is not None and isinstance(
- insecure_interagent_communication.simulated_attacks, dict
+ assert (
+ insecure_interagent_communication.simulated_attacks is not None
+ and isinstance(
+ insecure_interagent_communication.simulated_attacks, dict
+ )
+ )
+ assert insecure_interagent_communication.res is not None and isinstance(
+ insecure_interagent_communication.res, dict
+ )
+ assert (
+ InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE in results
+ )
+ assert (
+ len(
+ results[InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE]
+ )
+ == 1
)
- assert insecure_interagent_communication.res is not None and isinstance(insecure_interagent_communication.res, dict)
- assert InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE in results
- assert len(results[InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE]) == 1
- test_case = results[InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE][0]
+ test_case = results[
+ InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE
+ ][0]
assert hasattr(test_case, "score")
assert hasattr(test_case, "reason")
diff --git a/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py b/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py
index 093971eb..64904791 100644
--- a/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py
+++ b/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py
@@ -1,7 +1,9 @@
import pytest
from deepteam.vulnerabilities import ToolMetadataPoisoning
-from deepteam.vulnerabilities.tool_metadata_poisoning import ToolMetadataPoisoningType
+from deepteam.vulnerabilities.tool_metadata_poisoning import (
+ ToolMetadataPoisoningType,
+)
from deepteam.test_case import RTTestCase
@@ -15,33 +17,43 @@ class TestToolMetadataPoisoning:
"registry_poisoning",
]
tool_metadata_poisoning = ToolMetadataPoisoning(types=types)
- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_metadata_poisoning.types
+ ) == sorted(types)
def test_tool_metadata_poisoning_all_types_default(self):
tool_metadata_poisoning = ToolMetadataPoisoning()
- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(
- type.value for type in ToolMetadataPoisoningType
- )
+ assert sorted(
+ type.value for type in tool_metadata_poisoning.types
+ ) == sorted(type.value for type in ToolMetadataPoisoningType)
def test_tool_metadata_poisoning_schema_manipulation(self):
types = ["schema_manipulation"]
tool_metadata_poisoning = ToolMetadataPoisoning(types=types)
- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_metadata_poisoning.types
+ ) == sorted(types)
def test_tool_metadata_poisoning_description_deception(self):
types = ["description_deception"]
tool_metadata_poisoning = ToolMetadataPoisoning(types=types)
- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_metadata_poisoning.types
+ ) == sorted(types)
def test_tool_metadata_poisoning_permission_misrepresentation(self):
types = ["permission_misrepresentation"]
tool_metadata_poisoning = ToolMetadataPoisoning(types=types)
- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_metadata_poisoning.types
+ ) == sorted(types)
def test_tool_metadata_poisoning_registry_poisoning(self):
types = ["registry_poisoning"]
tool_metadata_poisoning = ToolMetadataPoisoning(types=types)
- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_metadata_poisoning.types
+ ) == sorted(types)
def test_tool_metadata_poisoning_all_types_invalid(self):
types = [
@@ -55,16 +67,21 @@ class TestToolMetadataPoisoning:
ToolMetadataPoisoning(types=types)
def test_simulate_attacks_returns_expected_cases(self):
- tool_metadata_poisoning = ToolMetadataPoisoning(types=["schema_manipulation"])
+ tool_metadata_poisoning = ToolMetadataPoisoning(
+ types=["schema_manipulation"]
+ )
test_cases = tool_metadata_poisoning.simulate_attacks(
attacks_per_vulnerability_type=2
)
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Tool Metadata Poisoning" for tc in test_cases)
assert all(
- tc.vulnerability_type == ToolMetadataPoisoningType.SCHEMA_MANIPULATION
+ tc.vulnerability == "Tool Metadata Poisoning" for tc in test_cases
+ )
+ assert all(
+ tc.vulnerability_type
+ == ToolMetadataPoisoningType.SCHEMA_MANIPULATION
for tc in test_cases
)
@@ -81,12 +98,17 @@ class TestToolMetadataPoisoning:
)
assert tool_metadata_poisoning.is_vulnerable() is not None
- assert tool_metadata_poisoning.simulated_attacks is not None and isinstance(
- tool_metadata_poisoning.simulated_attacks, dict
+ assert (
+ tool_metadata_poisoning.simulated_attacks is not None
+ and isinstance(tool_metadata_poisoning.simulated_attacks, dict)
+ )
+ assert tool_metadata_poisoning.res is not None and isinstance(
+ tool_metadata_poisoning.res, dict
)
- assert tool_metadata_poisoning.res is not None and isinstance(tool_metadata_poisoning.res, dict)
assert ToolMetadataPoisoningType.DESCRIPTION_DECEPTION in results
- assert len(results[ToolMetadataPoisoningType.DESCRIPTION_DECEPTION]) == 1
+ assert (
+ len(results[ToolMetadataPoisoningType.DESCRIPTION_DECEPTION]) == 1
+ )
test_case = results[ToolMetadataPoisoningType.DESCRIPTION_DECEPTION][0]
# The score and reason will depend on the real metric evaluation
assert hasattr(test_case, "score")
@@ -98,14 +120,18 @@ class TestToolMetadataPoisoning:
tool_metadata_poisoning = ToolMetadataPoisoning(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = tool_metadata_poisoning._get_metric(ToolMetadataPoisoningType.DESCRIPTION_DECEPTION)
+ metric = tool_metadata_poisoning._get_metric(
+ ToolMetadataPoisoningType.DESCRIPTION_DECEPTION
+ )
assert isinstance(metric, ToolMetadataPoisoningMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- tool_metadata_poisoning = ToolMetadataPoisoning(types=["registry_poisoning"], async_mode=True)
+ tool_metadata_poisoning = ToolMetadataPoisoning(
+ types=["registry_poisoning"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -115,10 +141,13 @@ class TestToolMetadataPoisoning:
)
assert tool_metadata_poisoning.is_vulnerable() is not None
- assert tool_metadata_poisoning.simulated_attacks is not None and isinstance(
- tool_metadata_poisoning.simulated_attacks, dict
+ assert (
+ tool_metadata_poisoning.simulated_attacks is not None
+ and isinstance(tool_metadata_poisoning.simulated_attacks, dict)
+ )
+ assert tool_metadata_poisoning.res is not None and isinstance(
+ tool_metadata_poisoning.res, dict
)
- assert tool_metadata_poisoning.res is not None and isinstance(tool_metadata_poisoning.res, dict)
assert ToolMetadataPoisoningType.REGISTRY_POISONING in results
assert len(results[ToolMetadataPoisoningType.REGISTRY_POISONING]) == 1
test_case = results[ToolMetadataPoisoningType.REGISTRY_POISONING][0]
diff --git a/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py b/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py
index 9daaa167..5cd288ca 100644
--- a/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py
+++ b/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py
@@ -1,7 +1,9 @@
import pytest
from deepteam.vulnerabilities import ToolOrchestrationAbuse
-from deepteam.vulnerabilities.tool_orchestration_abuse import ToolOrchestrationAbuseType
+from deepteam.vulnerabilities.tool_orchestration_abuse import (
+ ToolOrchestrationAbuseType,
+)
from deepteam.test_case import RTTestCase
@@ -15,33 +17,43 @@ class TestToolOrchestrationAbuse:
"cross_tool_state_leakage",
]
tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)
- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_orchestration_abuse.types
+ ) == sorted(types)
def test_tool_orchestration_abuse_all_types_default(self):
tool_orchestration_abuse = ToolOrchestrationAbuse()
- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(
- type.value for type in ToolOrchestrationAbuseType
- )
+ assert sorted(
+ type.value for type in tool_orchestration_abuse.types
+ ) == sorted(type.value for type in ToolOrchestrationAbuseType)
def test_tool_orchestration_abuse_recursive_tool_calls(self):
types = ["recursive_tool_calls"]
tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)
- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_orchestration_abuse.types
+ ) == sorted(types)
def test_tool_orchestration_abuse_unsafe_tool_composition(self):
types = ["unsafe_tool_composition"]
tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)
- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_orchestration_abuse.types
+ ) == sorted(types)
def test_tool_orchestration_abuse_tool_budget_exhaustion(self):
types = ["tool_budget_exhaustion"]
tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)
- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_orchestration_abuse.types
+ ) == sorted(types)
def test_tool_orchestration_abuse_cross_tool_state_leakage(self):
types = ["cross_tool_state_leakage"]
tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)
- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)
+ assert sorted(
+ type.value for type in tool_orchestration_abuse.types
+ ) == sorted(types)
def test_tool_orchestration_abuse_all_types_invalid(self):
types = [
@@ -55,16 +67,21 @@ class TestToolOrchestrationAbuse:
ToolOrchestrationAbuse(types=types)
def test_simulate_attacks_returns_expected_cases(self):
- tool_orchestration_abuse = ToolOrchestrationAbuse(types=["recursive_tool_calls"])
+ tool_orchestration_abuse = ToolOrchestrationAbuse(
+ types=["recursive_tool_calls"]
+ )
test_cases = tool_orchestration_abuse.simulate_attacks(
attacks_per_vulnerability_type=2
)
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Tool Orchestration Abuse" for tc in test_cases)
assert all(
- tc.vulnerability_type == ToolOrchestrationAbuseType.RECURSIVE_TOOL_CALLS
+ tc.vulnerability == "Tool Orchestration Abuse" for tc in test_cases
+ )
+ assert all(
+ tc.vulnerability_type
+ == ToolOrchestrationAbuseType.RECURSIVE_TOOL_CALLS
for tc in test_cases
)
@@ -81,13 +98,21 @@ class TestToolOrchestrationAbuse:
)
assert tool_orchestration_abuse.is_vulnerable() is not None
- assert tool_orchestration_abuse.simulated_attacks is not None and isinstance(
- tool_orchestration_abuse.simulated_attacks, dict
+ assert (
+ tool_orchestration_abuse.simulated_attacks is not None
+ and isinstance(tool_orchestration_abuse.simulated_attacks, dict)
+ )
+ assert tool_orchestration_abuse.res is not None and isinstance(
+ tool_orchestration_abuse.res, dict
)
- assert tool_orchestration_abuse.res is not None and isinstance(tool_orchestration_abuse.res, dict)
assert ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION in results
- assert len(results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION]) == 1
- test_case = results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION][0]
+ assert (
+ len(results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION])
+ == 1
+ )
+ test_case = results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION][
+ 0
+ ]
# The score and reason will depend on the real metric evaluation
assert hasattr(test_case, "score")
assert hasattr(test_case, "reason")
@@ -98,14 +123,18 @@ class TestToolOrchestrationAbuse:
tool_orchestration_abuse = ToolOrchestrationAbuse(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = tool_orchestration_abuse._get_metric(ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION)
+ metric = tool_orchestration_abuse._get_metric(
+ ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION
+ )
assert isinstance(metric, ToolOrchestrationMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- tool_orchestration_abuse = ToolOrchestrationAbuse(types=["cross_tool_state_leakage"], async_mode=True)
+ tool_orchestration_abuse = ToolOrchestrationAbuse(
+ types=["cross_tool_state_leakage"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -115,12 +144,20 @@ class TestToolOrchestrationAbuse:
)
assert tool_orchestration_abuse.is_vulnerable() is not None
- assert tool_orchestration_abuse.simulated_attacks is not None and isinstance(
- tool_orchestration_abuse.simulated_attacks, dict
+ assert (
+ tool_orchestration_abuse.simulated_attacks is not None
+ and isinstance(tool_orchestration_abuse.simulated_attacks, dict)
+ )
+ assert tool_orchestration_abuse.res is not None and isinstance(
+ tool_orchestration_abuse.res, dict
)
- assert tool_orchestration_abuse.res is not None and isinstance(tool_orchestration_abuse.res, dict)
assert ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE in results
- assert len(results[ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE]) == 1
- test_case = results[ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE][0]
+ assert (
+ len(results[ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE])
+ == 1
+ )
+ test_case = results[
+ ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE
+ ][0]
assert hasattr(test_case, "score")
assert hasattr(test_case, "reason")
diff --git a/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py b/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py
index bb78f45b..ed4a5b01 100644
--- a/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py
+++ b/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py
@@ -1,7 +1,9 @@
import pytest
from deepteam.vulnerabilities import UnexpectedCodeExecution
-from deepteam.vulnerabilities.unexpected_code_execution import UnexpectedCodeExecutionType
+from deepteam.vulnerabilities.unexpected_code_execution import (
+ UnexpectedCodeExecutionType,
+)
from deepteam.test_case import RTTestCase
@@ -14,28 +16,36 @@ class TestUnexpectedCodeExecution:
"eval_usage",
]
unexpected_code_execution = UnexpectedCodeExecution(types=types)
- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)
+ assert sorted(
+ type.value for type in unexpected_code_execution.types
+ ) == sorted(types)
def test_unexpected_code_execution_all_types_default(self):
unexpected_code_execution = UnexpectedCodeExecution()
- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(
- type.value for type in UnexpectedCodeExecutionType
- )
+ assert sorted(
+ type.value for type in unexpected_code_execution.types
+ ) == sorted(type.value for type in UnexpectedCodeExecutionType)
def test_unexpected_code_execution_unauthorized_code_execution(self):
types = ["unauthorized_code_execution"]
unexpected_code_execution = UnexpectedCodeExecution(types=types)
- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)
+ assert sorted(
+ type.value for type in unexpected_code_execution.types
+ ) == sorted(types)
def test_unexpected_code_execution_shell_command_execution(self):
types = ["shell_command_execution"]
unexpected_code_execution = UnexpectedCodeExecution(types=types)
- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)
+ assert sorted(
+ type.value for type in unexpected_code_execution.types
+ ) == sorted(types)
def test_unexpected_code_execution_eval_usage(self):
types = ["eval_usage"]
unexpected_code_execution = UnexpectedCodeExecution(types=types)
- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)
+ assert sorted(
+ type.value for type in unexpected_code_execution.types
+ ) == sorted(types)
def test_unexpected_code_execution_all_types_invalid(self):
types = [
@@ -48,16 +58,21 @@ class TestUnexpectedCodeExecution:
UnexpectedCodeExecution(types=types)
def test_simulate_attacks_returns_expected_cases(self):
- unexpected_code_execution = UnexpectedCodeExecution(types=["unauthorized_code_execution"])
+ unexpected_code_execution = UnexpectedCodeExecution(
+ types=["unauthorized_code_execution"]
+ )
test_cases = unexpected_code_execution.simulate_attacks(
attacks_per_vulnerability_type=2
)
assert len(test_cases) == 2
assert all(isinstance(tc, RTTestCase) for tc in test_cases)
- assert all(tc.vulnerability == "Unexpected Code Execution" for tc in test_cases)
assert all(
- tc.vulnerability_type == UnexpectedCodeExecutionType.UNAUTHORIZED_CODE_EXECUTION
+ tc.vulnerability == "Unexpected Code Execution" for tc in test_cases
+ )
+ assert all(
+ tc.vulnerability_type
+ == UnexpectedCodeExecutionType.UNAUTHORIZED_CODE_EXECUTION
for tc in test_cases
)
@@ -74,13 +89,21 @@ class TestUnexpectedCodeExecution:
)
assert unexpected_code_execution.is_vulnerable() is not None
- assert unexpected_code_execution.simulated_attacks is not None and isinstance(
- unexpected_code_execution.simulated_attacks, dict
+ assert (
+ unexpected_code_execution.simulated_attacks is not None
+ and isinstance(unexpected_code_execution.simulated_attacks, dict)
+ )
+ assert unexpected_code_execution.res is not None and isinstance(
+ unexpected_code_execution.res, dict
)
- assert unexpected_code_execution.res is not None and isinstance(unexpected_code_execution.res, dict)
assert UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION in results
- assert len(results[UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION]) == 1
- test_case = results[UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION][0]
+ assert (
+ len(results[UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION])
+ == 1
+ )
+ test_case = results[
+ UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION
+ ][0]
# The score and reason will depend on the real metric evaluation
assert hasattr(test_case, "score")
assert hasattr(test_case, "reason")
@@ -91,14 +114,18 @@ class TestUnexpectedCodeExecution:
unexpected_code_execution = UnexpectedCodeExecution(
async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"
)
- metric = unexpected_code_execution._get_metric(UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION)
+ metric = unexpected_code_execution._get_metric(
+ UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION
+ )
assert isinstance(metric, UnexpectedCodeExecutionMetric)
assert metric.async_mode is True
assert metric.verbose_mode is True
@pytest.mark.asyncio
async def test_a_assess_returns_async_results(self):
- unexpected_code_execution = UnexpectedCodeExecution(types=["eval_usage"], async_mode=True)
+ unexpected_code_execution = UnexpectedCodeExecution(
+ types=["eval_usage"], async_mode=True
+ )
async def dummy_model_callback(prompt):
return prompt
@@ -108,10 +135,13 @@ class TestUnexpectedCodeExecution:
)
assert unexpected_code_execution.is_vulnerable() is not None
- assert unexpected_code_execution.simulated_attacks is not None and isinstance(
- unexpected_code_execution.simulated_attacks, dict
+ assert (
+ unexpected_code_execution.simulated_attacks is not None
+ and isinstance(unexpected_code_execution.simulated_attacks, dict)
+ )
+ assert unexpected_code_execution.res is not None and isinstance(
+ unexpected_code_execution.res, dict
)
- assert unexpected_code_execution.res is not None and isinstance(unexpected_code_execution.res, dict)
assert UnexpectedCodeExecutionType.EVAL_USAGE in results
assert len(results[UnexpectedCodeExecutionType.EVAL_USAGE]) == 1
test_case = results[UnexpectedCodeExecutionType.EVAL_USAGE][0]
diff --git a/tests/test_frameworks/test_owasp_top_10_asi.py b/tests/test_frameworks/test_owasp_top_10_asi.py
index 64e2d2f4..df11f492 100644
--- a/tests/test_frameworks/test_owasp_top_10_asi.py
+++ b/tests/test_frameworks/test_owasp_top_10_asi.py
@@ -3,7 +3,9 @@ from deepteam.frameworks import OWASP_ASI_2026
from deepteam.red_teamer.risk_assessment import RiskAssessment
from deepteam.vulnerabilities import BaseVulnerability
from deepteam.attacks import BaseAttack
-from deepteam.frameworks.owasp_top_10_agentic.risk_categories import OWASP_ASI_CATEGORIES
+from deepteam.frameworks.owasp_top_10_agentic.risk_categories import (
+ OWASP_ASI_CATEGORIES,
+)
from deepteam.frameworks.risk_category import RiskCategory
from deepteam import red_team