DeepTeam is a framework to red team LLMs and AI agents.
lint
A-Vamshi037da30b8956
diff --git a/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py b/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.pyindex ae2a200b..829a68f6 100644--- a/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py+++ b/tests/test_core/test_vulnerabilities/test_agent_identity_abuse.py@@ -14,28 +14,36 @@ class TestAgentIdentityAbuse:"cross_agent_trust_abuse",]agent_identity_abuse = AgentIdentityAbuse(types=types)- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in agent_identity_abuse.types+ ) == sorted(types)def test_agent_identity_abuse_all_types_default(self):agent_identity_abuse = AgentIdentityAbuse()- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(- type.value for type in AgentIdentityAbuseType- )+ assert sorted(+ type.value for type in agent_identity_abuse.types+ ) == sorted(type.value for type in AgentIdentityAbuseType)def test_agent_identity_abuse_agent_impersonation(self):types = ["agent_impersonation"]agent_identity_abuse = AgentIdentityAbuse(types=types)- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in agent_identity_abuse.types+ ) == sorted(types)def test_agent_identity_abuse_identity_inheritance(self):types = ["identity_inheritance"]agent_identity_abuse = AgentIdentityAbuse(types=types)- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in agent_identity_abuse.types+ ) == sorted(types)def test_agent_identity_abuse_cross_agent_trust_abuse(self):types = ["cross_agent_trust_abuse"]agent_identity_abuse = AgentIdentityAbuse(types=types)- assert sorted(type.value for type in agent_identity_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in agent_identity_abuse.types+ ) == sorted(types)def test_agent_identity_abuse_all_types_invalid(self):types = [@@ -55,7 +63,10 @@ class TestAgentIdentityAbuse:assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Agent Identity & Trust Abuse" for tc in test_cases)+ assert all(+ tc.vulnerability == "Agent Identity & Trust Abuse"+ for tc in test_cases+ )assert all(tc.vulnerability_type == AgentIdentityAbuseType.AGENT_IMPERSONATIONfor tc in test_cases@@ -74,10 +85,13 @@ class TestAgentIdentityAbuse:)assert agent_identity_abuse.is_vulnerable() is not None- assert agent_identity_abuse.simulated_attacks is not None and isinstance(- agent_identity_abuse.simulated_attacks, dict+ assert (+ agent_identity_abuse.simulated_attacks is not None+ and isinstance(agent_identity_abuse.simulated_attacks, dict)+ )+ assert agent_identity_abuse.res is not None and isinstance(+ agent_identity_abuse.res, dict)- assert agent_identity_abuse.res is not None and isinstance(agent_identity_abuse.res, dict)assert AgentIdentityAbuseType.IDENTITY_INHERITANCE in resultsassert len(results[AgentIdentityAbuseType.IDENTITY_INHERITANCE]) == 1test_case = results[AgentIdentityAbuseType.IDENTITY_INHERITANCE][0]@@ -91,14 +105,18 @@ class TestAgentIdentityAbuse:agent_identity_abuse = AgentIdentityAbuse(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = agent_identity_abuse._get_metric(AgentIdentityAbuseType.IDENTITY_INHERITANCE)+ metric = agent_identity_abuse._get_metric(+ AgentIdentityAbuseType.IDENTITY_INHERITANCE+ )assert isinstance(metric, AgentIdentityAbuseMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- agent_identity_abuse = AgentIdentityAbuse(types=["cross_agent_trust_abuse"], async_mode=True)+ agent_identity_abuse = AgentIdentityAbuse(+ types=["cross_agent_trust_abuse"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -108,10 +126,13 @@ class TestAgentIdentityAbuse:)assert agent_identity_abuse.is_vulnerable() is not None- assert agent_identity_abuse.simulated_attacks is not None and isinstance(- agent_identity_abuse.simulated_attacks, dict+ assert (+ agent_identity_abuse.simulated_attacks is not None+ and isinstance(agent_identity_abuse.simulated_attacks, dict)+ )+ assert agent_identity_abuse.res is not None and isinstance(+ agent_identity_abuse.res, dict)- assert agent_identity_abuse.res is not None and isinstance(agent_identity_abuse.res, dict)assert AgentIdentityAbuseType.CROSS_AGENT_TRUST_ABUSE in resultsassert len(results[AgentIdentityAbuseType.CROSS_AGENT_TRUST_ABUSE]) == 1test_case = results[AgentIdentityAbuseType.CROSS_AGENT_TRUST_ABUSE][0]diff --git a/tests/test_core/test_vulnerabilities/test_all.py b/tests/test_core/test_vulnerabilities/test_all.pyindex 8535330c..3ac3ad23 100644--- a/tests/test_core/test_vulnerabilities/test_all.py+++ b/tests/test_core/test_vulnerabilities/test_all.py@@ -48,7 +48,7 @@ def test_red_teaming():ToolMetadataPoisoning(),UnexpectedCodeExecution(),InsecureInterAgentCommunication(),- AutonomousAgentDrift(),+ AutonomousAgentDrift(),custom_vulnerability,]diff --git a/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py b/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.pyindex ee9dee3c..1758aeee 100644--- a/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py+++ b/tests/test_core/test_vulnerabilities/test_autonomous_agent_drift.py@@ -1,7 +1,9 @@import pytestfrom deepteam.vulnerabilities import AutonomousAgentDrift-from deepteam.vulnerabilities.autonomous_agent_drift import AutonomousAgentDriftType+from deepteam.vulnerabilities.autonomous_agent_drift import (+ AutonomousAgentDriftType,+)from deepteam.test_case import RTTestCase@@ -15,33 +17,43 @@ class TestAutonomousAgentDrift:"runaway_autonomy",]autonoumus_agent_drift = AutonomousAgentDrift(types=types)- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)+ assert sorted(+ type.value for type in autonoumus_agent_drift.types+ ) == sorted(types)def test_autonomous_agent_drift_all_types_default(self):autonoumus_agent_drift = AutonomousAgentDrift()- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(- type.value for type in AutonomousAgentDriftType- )+ assert sorted(+ type.value for type in autonoumus_agent_drift.types+ ) == sorted(type.value for type in AutonomousAgentDriftType)def test_autonomous_agent_drift_goal_drift(self):types = ["goal_drift"]autonoumus_agent_drift = AutonomousAgentDrift(types=types)- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)+ assert sorted(+ type.value for type in autonoumus_agent_drift.types+ ) == sorted(types)def test_autonomous_agent_drift_reward_hacking(self):types = ["reward_hacking"]autonoumus_agent_drift = AutonomousAgentDrift(types=types)- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)+ assert sorted(+ type.value for type in autonoumus_agent_drift.types+ ) == sorted(types)def test_autonomous_agent_drift_agent_collusion(self):types = ["agent_collusion"]autonoumus_agent_drift = AutonomousAgentDrift(types=types)- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)+ assert sorted(+ type.value for type in autonoumus_agent_drift.types+ ) == sorted(types)def test_autonomous_agent_drift_runaway_autonomy(self):types = ["runaway_autonomy"]autonoumus_agent_drift = AutonomousAgentDrift(types=types)- assert sorted(type.value for type in autonoumus_agent_drift.types) == sorted(types)+ assert sorted(+ type.value for type in autonoumus_agent_drift.types+ ) == sorted(types)def test_autonomous_agent_drift_all_types_invalid(self):types = [@@ -62,7 +74,9 @@ class TestAutonomousAgentDrift:assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Autonomous Agent Drift" for tc in test_cases)+ assert all(+ tc.vulnerability == "Autonomous Agent Drift" for tc in test_cases+ )assert all(tc.vulnerability_type == AutonomousAgentDriftType.GOAL_DRIFTfor tc in test_cases@@ -81,10 +95,13 @@ class TestAutonomousAgentDrift:)assert autonoumus_agent_drift.is_vulnerable() is not None- assert autonoumus_agent_drift.simulated_attacks is not None and isinstance(- autonoumus_agent_drift.simulated_attacks, dict+ assert (+ autonoumus_agent_drift.simulated_attacks is not None+ and isinstance(autonoumus_agent_drift.simulated_attacks, dict)+ )+ assert autonoumus_agent_drift.res is not None and isinstance(+ autonoumus_agent_drift.res, dict)- assert autonoumus_agent_drift.res is not None and isinstance(autonoumus_agent_drift.res, dict)assert AutonomousAgentDriftType.REWARD_HACKING in resultsassert len(results[AutonomousAgentDriftType.REWARD_HACKING]) == 1test_case = results[AutonomousAgentDriftType.REWARD_HACKING][0]@@ -98,14 +115,18 @@ class TestAutonomousAgentDrift:autonoumus_agent_drift = AutonomousAgentDrift(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = autonoumus_agent_drift._get_metric(AutonomousAgentDriftType.REWARD_HACKING)+ metric = autonoumus_agent_drift._get_metric(+ AutonomousAgentDriftType.REWARD_HACKING+ )assert isinstance(metric, AutonomousAgentDriftMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- autonoumus_agent_drift = AutonomousAgentDrift(types=["runaway_autonomy"], async_mode=True)+ autonoumus_agent_drift = AutonomousAgentDrift(+ types=["runaway_autonomy"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -115,10 +136,13 @@ class TestAutonomousAgentDrift:)assert autonoumus_agent_drift.is_vulnerable() is not None- assert autonoumus_agent_drift.simulated_attacks is not None and isinstance(- autonoumus_agent_drift.simulated_attacks, dict+ assert (+ autonoumus_agent_drift.simulated_attacks is not None+ and isinstance(autonoumus_agent_drift.simulated_attacks, dict)+ )+ assert autonoumus_agent_drift.res is not None and isinstance(+ autonoumus_agent_drift.res, dict)- assert autonoumus_agent_drift.res is not None and isinstance(autonoumus_agent_drift.res, dict)assert AutonomousAgentDriftType.RUNAWAY_AUTONOMY in resultsassert len(results[AutonomousAgentDriftType.RUNAWAY_AUTONOMY]) == 1test_case = results[AutonomousAgentDriftType.RUNAWAY_AUTONOMY][0]diff --git a/tests/test_core/test_vulnerabilities/test_indirect_instruction.py b/tests/test_core/test_vulnerabilities/test_indirect_instruction.pyindex aaa13482..84efabb4 100644--- a/tests/test_core/test_vulnerabilities/test_indirect_instruction.py+++ b/tests/test_core/test_vulnerabilities/test_indirect_instruction.py@@ -1,7 +1,9 @@import pytestfrom deepteam.vulnerabilities import IndirectInstruction-from deepteam.vulnerabilities.indirect_instruction import IndirectInstructionType+from deepteam.vulnerabilities.indirect_instruction import (+ IndirectInstructionType,+)from deepteam.test_case import RTTestCase@@ -15,33 +17,43 @@ class TestIndirectInstruction:"cross_context_injection",]indirect_instruction = IndirectInstruction(types=types)- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)+ assert sorted(+ type.value for type in indirect_instruction.types+ ) == sorted(types)def test_indirect_instruction_all_types_default(self):indirect_instruction = IndirectInstruction()- assert sorted(type.value for type in indirect_instruction.types) == sorted(- type.value for type in IndirectInstructionType- )+ assert sorted(+ type.value for type in indirect_instruction.types+ ) == sorted(type.value for type in IndirectInstructionType)def test_indirect_instruction_rag_injection(self):types = ["rag_injection"]indirect_instruction = IndirectInstruction(types=types)- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)+ assert sorted(+ type.value for type in indirect_instruction.types+ ) == sorted(types)def test_indirect_instruction_tool_output_injection(self):types = ["tool_output_injection"]indirect_instruction = IndirectInstruction(types=types)- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)+ assert sorted(+ type.value for type in indirect_instruction.types+ ) == sorted(types)def test_indirect_instruction_document_embedded_instructions(self):types = ["document_embedded_instructions"]indirect_instruction = IndirectInstruction(types=types)- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)+ assert sorted(+ type.value for type in indirect_instruction.types+ ) == sorted(types)def test_indirect_instruction_cross_context_injection(self):types = ["cross_context_injection"]indirect_instruction = IndirectInstruction(types=types)- assert sorted(type.value for type in indirect_instruction.types) == sorted(types)+ assert sorted(+ type.value for type in indirect_instruction.types+ ) == sorted(types)def test_indirect_instruction_all_types_invalid(self):types = [@@ -62,7 +74,9 @@ class TestIndirectInstruction:assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Indirect Instruction" for tc in test_cases)+ assert all(+ tc.vulnerability == "Indirect Instruction" for tc in test_cases+ )assert all(tc.vulnerability_type == IndirectInstructionType.RAG_INJECTIONfor tc in test_cases@@ -81,10 +95,13 @@ class TestIndirectInstruction:)assert indirect_instruction.is_vulnerable() is not None- assert indirect_instruction.simulated_attacks is not None and isinstance(- indirect_instruction.simulated_attacks, dict+ assert (+ indirect_instruction.simulated_attacks is not None+ and isinstance(indirect_instruction.simulated_attacks, dict)+ )+ assert indirect_instruction.res is not None and isinstance(+ indirect_instruction.res, dict)- assert indirect_instruction.res is not None and isinstance(indirect_instruction.res, dict)assert IndirectInstructionType.TOOL_OUTPUT_INJECTION in resultsassert len(results[IndirectInstructionType.TOOL_OUTPUT_INJECTION]) == 1test_case = results[IndirectInstructionType.TOOL_OUTPUT_INJECTION][0]@@ -98,14 +115,18 @@ class TestIndirectInstruction:indirect_instruction = IndirectInstruction(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = indirect_instruction._get_metric(IndirectInstructionType.TOOL_OUTPUT_INJECTION)+ metric = indirect_instruction._get_metric(+ IndirectInstructionType.TOOL_OUTPUT_INJECTION+ )assert isinstance(metric, IndirectInstructionMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- indirect_instruction = IndirectInstruction(types=["cross_context_injection"], async_mode=True)+ indirect_instruction = IndirectInstruction(+ types=["cross_context_injection"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -115,12 +136,17 @@ class TestIndirectInstruction:)assert indirect_instruction.is_vulnerable() is not None- assert indirect_instruction.simulated_attacks is not None and isinstance(- indirect_instruction.simulated_attacks, dict+ assert (+ indirect_instruction.simulated_attacks is not None+ and isinstance(indirect_instruction.simulated_attacks, dict)+ )+ assert indirect_instruction.res is not None and isinstance(+ indirect_instruction.res, dict)- assert indirect_instruction.res is not None and isinstance(indirect_instruction.res, dict)assert IndirectInstructionType.CROSS_CONTENT_INJECTION in results- assert len(results[IndirectInstructionType.CROSS_CONTENT_INJECTION]) == 1+ assert (+ len(results[IndirectInstructionType.CROSS_CONTENT_INJECTION]) == 1+ )test_case = results[IndirectInstructionType.CROSS_CONTENT_INJECTION][0]assert hasattr(test_case, "score")assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py b/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.pyindex 9e60038b..5ce13d97 100644--- a/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py+++ b/tests/test_core/test_vulnerabilities/test_insecure_interagent_communication.py@@ -1,7 +1,9 @@import pytestfrom deepteam.vulnerabilities import InsecureInterAgentCommunication-from deepteam.vulnerabilities.insecure_inter_agent_communication import InsecureInterAgentCommunicationType+from deepteam.vulnerabilities.insecure_inter_agent_communication import (+ InsecureInterAgentCommunicationType,+)from deepteam.test_case import RTTestCase@@ -13,29 +15,45 @@ class TestInsecureInterAgentCommunication:"message_injection","agent_in_the_middle",]- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)+ insecure_interagent_communication = InsecureInterAgentCommunication(+ types=types+ )+ assert sorted(+ type.value for type in insecure_interagent_communication.types+ ) == sorted(types)def test_insecure_interagent_communication_all_types_default(self):insecure_interagent_communication = InsecureInterAgentCommunication()- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(- type.value for type in InsecureInterAgentCommunicationType- )+ assert sorted(+ type.value for type in insecure_interagent_communication.types+ ) == sorted(type.value for type in InsecureInterAgentCommunicationType)def test_insecure_interagent_communication_message_spoofing(self):types = ["message_spoofing"]- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)+ insecure_interagent_communication = InsecureInterAgentCommunication(+ types=types+ )+ assert sorted(+ type.value for type in insecure_interagent_communication.types+ ) == sorted(types)def test_insecure_interagent_communication_message_injection(self):types = ["message_injection"]- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)+ insecure_interagent_communication = InsecureInterAgentCommunication(+ types=types+ )+ assert sorted(+ type.value for type in insecure_interagent_communication.types+ ) == sorted(types)def test_insecure_interagent_communication_agent_in_the_middle(self):types = ["agent_in_the_middle"]- insecure_interagent_communication = InsecureInterAgentCommunication(types=types)- assert sorted(type.value for type in insecure_interagent_communication.types) == sorted(types)+ insecure_interagent_communication = InsecureInterAgentCommunication(+ types=types+ )+ assert sorted(+ type.value for type in insecure_interagent_communication.types+ ) == sorted(types)def test_insecure_interagent_communication_all_types_invalid(self):types = [@@ -48,16 +66,22 @@ class TestInsecureInterAgentCommunication:InsecureInterAgentCommunication(types=types)def test_simulate_attacks_returns_expected_cases(self):- insecure_interagent_communication = InsecureInterAgentCommunication(types=["message_spoofing"])+ insecure_interagent_communication = InsecureInterAgentCommunication(+ types=["message_spoofing"]+ )test_cases = insecure_interagent_communication.simulate_attacks(attacks_per_vulnerability_type=2)assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Inter-Agent Communication Compromise" for tc in test_cases)assert all(- tc.vulnerability_type == InsecureInterAgentCommunicationType.MESSAGE_SPOOFING+ tc.vulnerability == "Inter-Agent Communication Compromise"+ for tc in test_cases+ )+ assert all(+ tc.vulnerability_type+ == InsecureInterAgentCommunicationType.MESSAGE_SPOOFINGfor tc in test_cases)@@ -74,13 +98,23 @@ class TestInsecureInterAgentCommunication:)assert insecure_interagent_communication.is_vulnerable() is not None- assert insecure_interagent_communication.simulated_attacks is not None and isinstance(- insecure_interagent_communication.simulated_attacks, dict+ assert (+ insecure_interagent_communication.simulated_attacks is not None+ and isinstance(+ insecure_interagent_communication.simulated_attacks, dict+ )+ )+ assert insecure_interagent_communication.res is not None and isinstance(+ insecure_interagent_communication.res, dict)- assert insecure_interagent_communication.res is not None and isinstance(insecure_interagent_communication.res, dict)assert InsecureInterAgentCommunicationType.MESSAGE_INJECTION in results- assert len(results[InsecureInterAgentCommunicationType.MESSAGE_INJECTION]) == 1- test_case = results[InsecureInterAgentCommunicationType.MESSAGE_INJECTION][0]+ assert (+ len(results[InsecureInterAgentCommunicationType.MESSAGE_INJECTION])+ == 1+ )+ test_case = results[+ InsecureInterAgentCommunicationType.MESSAGE_INJECTION+ ][0]# The score and reason will depend on the real metric evaluationassert hasattr(test_case, "score")assert hasattr(test_case, "reason")@@ -91,14 +125,18 @@ class TestInsecureInterAgentCommunication:insecure_interagent_communication = InsecureInterAgentCommunication(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = insecure_interagent_communication._get_metric(InsecureInterAgentCommunicationType.MESSAGE_INJECTION)+ metric = insecure_interagent_communication._get_metric(+ InsecureInterAgentCommunicationType.MESSAGE_INJECTION+ )assert isinstance(metric, InsecureInterAgentCommunicationMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- insecure_interagent_communication = InsecureInterAgentCommunication(types=["agent_in_the_middle"], async_mode=True)+ insecure_interagent_communication = InsecureInterAgentCommunication(+ types=["agent_in_the_middle"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -108,12 +146,26 @@ class TestInsecureInterAgentCommunication:)assert insecure_interagent_communication.is_vulnerable() is not None- assert insecure_interagent_communication.simulated_attacks is not None and isinstance(- insecure_interagent_communication.simulated_attacks, dict+ assert (+ insecure_interagent_communication.simulated_attacks is not None+ and isinstance(+ insecure_interagent_communication.simulated_attacks, dict+ )+ )+ assert insecure_interagent_communication.res is not None and isinstance(+ insecure_interagent_communication.res, dict+ )+ assert (+ InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE in results+ )+ assert (+ len(+ results[InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE]+ )+ == 1)- assert insecure_interagent_communication.res is not None and isinstance(insecure_interagent_communication.res, dict)- assert InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE in results- assert len(results[InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE]) == 1- test_case = results[InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE][0]+ test_case = results[+ InsecureInterAgentCommunicationType.AGENT_IN_THE_MIDDLE+ ][0]assert hasattr(test_case, "score")assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py b/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.pyindex 093971eb..64904791 100644--- a/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py+++ b/tests/test_core/test_vulnerabilities/test_tool_metadata_poisoning.py@@ -1,7 +1,9 @@import pytestfrom deepteam.vulnerabilities import ToolMetadataPoisoning-from deepteam.vulnerabilities.tool_metadata_poisoning import ToolMetadataPoisoningType+from deepteam.vulnerabilities.tool_metadata_poisoning import (+ ToolMetadataPoisoningType,+)from deepteam.test_case import RTTestCase@@ -15,33 +17,43 @@ class TestToolMetadataPoisoning:"registry_poisoning",]tool_metadata_poisoning = ToolMetadataPoisoning(types=types)- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)+ assert sorted(+ type.value for type in tool_metadata_poisoning.types+ ) == sorted(types)def test_tool_metadata_poisoning_all_types_default(self):tool_metadata_poisoning = ToolMetadataPoisoning()- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(- type.value for type in ToolMetadataPoisoningType- )+ assert sorted(+ type.value for type in tool_metadata_poisoning.types+ ) == sorted(type.value for type in ToolMetadataPoisoningType)def test_tool_metadata_poisoning_schema_manipulation(self):types = ["schema_manipulation"]tool_metadata_poisoning = ToolMetadataPoisoning(types=types)- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)+ assert sorted(+ type.value for type in tool_metadata_poisoning.types+ ) == sorted(types)def test_tool_metadata_poisoning_description_deception(self):types = ["description_deception"]tool_metadata_poisoning = ToolMetadataPoisoning(types=types)- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)+ assert sorted(+ type.value for type in tool_metadata_poisoning.types+ ) == sorted(types)def test_tool_metadata_poisoning_permission_misrepresentation(self):types = ["permission_misrepresentation"]tool_metadata_poisoning = ToolMetadataPoisoning(types=types)- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)+ assert sorted(+ type.value for type in tool_metadata_poisoning.types+ ) == sorted(types)def test_tool_metadata_poisoning_registry_poisoning(self):types = ["registry_poisoning"]tool_metadata_poisoning = ToolMetadataPoisoning(types=types)- assert sorted(type.value for type in tool_metadata_poisoning.types) == sorted(types)+ assert sorted(+ type.value for type in tool_metadata_poisoning.types+ ) == sorted(types)def test_tool_metadata_poisoning_all_types_invalid(self):types = [@@ -55,16 +67,21 @@ class TestToolMetadataPoisoning:ToolMetadataPoisoning(types=types)def test_simulate_attacks_returns_expected_cases(self):- tool_metadata_poisoning = ToolMetadataPoisoning(types=["schema_manipulation"])+ tool_metadata_poisoning = ToolMetadataPoisoning(+ types=["schema_manipulation"]+ )test_cases = tool_metadata_poisoning.simulate_attacks(attacks_per_vulnerability_type=2)assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Tool Metadata Poisoning" for tc in test_cases)assert all(- tc.vulnerability_type == ToolMetadataPoisoningType.SCHEMA_MANIPULATION+ tc.vulnerability == "Tool Metadata Poisoning" for tc in test_cases+ )+ assert all(+ tc.vulnerability_type+ == ToolMetadataPoisoningType.SCHEMA_MANIPULATIONfor tc in test_cases)@@ -81,12 +98,17 @@ class TestToolMetadataPoisoning:)assert tool_metadata_poisoning.is_vulnerable() is not None- assert tool_metadata_poisoning.simulated_attacks is not None and isinstance(- tool_metadata_poisoning.simulated_attacks, dict+ assert (+ tool_metadata_poisoning.simulated_attacks is not None+ and isinstance(tool_metadata_poisoning.simulated_attacks, dict)+ )+ assert tool_metadata_poisoning.res is not None and isinstance(+ tool_metadata_poisoning.res, dict)- assert tool_metadata_poisoning.res is not None and isinstance(tool_metadata_poisoning.res, dict)assert ToolMetadataPoisoningType.DESCRIPTION_DECEPTION in results- assert len(results[ToolMetadataPoisoningType.DESCRIPTION_DECEPTION]) == 1+ assert (+ len(results[ToolMetadataPoisoningType.DESCRIPTION_DECEPTION]) == 1+ )test_case = results[ToolMetadataPoisoningType.DESCRIPTION_DECEPTION][0]# The score and reason will depend on the real metric evaluationassert hasattr(test_case, "score")@@ -98,14 +120,18 @@ class TestToolMetadataPoisoning:tool_metadata_poisoning = ToolMetadataPoisoning(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = tool_metadata_poisoning._get_metric(ToolMetadataPoisoningType.DESCRIPTION_DECEPTION)+ metric = tool_metadata_poisoning._get_metric(+ ToolMetadataPoisoningType.DESCRIPTION_DECEPTION+ )assert isinstance(metric, ToolMetadataPoisoningMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- tool_metadata_poisoning = ToolMetadataPoisoning(types=["registry_poisoning"], async_mode=True)+ tool_metadata_poisoning = ToolMetadataPoisoning(+ types=["registry_poisoning"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -115,10 +141,13 @@ class TestToolMetadataPoisoning:)assert tool_metadata_poisoning.is_vulnerable() is not None- assert tool_metadata_poisoning.simulated_attacks is not None and isinstance(- tool_metadata_poisoning.simulated_attacks, dict+ assert (+ tool_metadata_poisoning.simulated_attacks is not None+ and isinstance(tool_metadata_poisoning.simulated_attacks, dict)+ )+ assert tool_metadata_poisoning.res is not None and isinstance(+ tool_metadata_poisoning.res, dict)- assert tool_metadata_poisoning.res is not None and isinstance(tool_metadata_poisoning.res, dict)assert ToolMetadataPoisoningType.REGISTRY_POISONING in resultsassert len(results[ToolMetadataPoisoningType.REGISTRY_POISONING]) == 1test_case = results[ToolMetadataPoisoningType.REGISTRY_POISONING][0]diff --git a/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py b/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.pyindex 9daaa167..5cd288ca 100644--- a/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py+++ b/tests/test_core/test_vulnerabilities/test_tool_orchestration_abuse.py@@ -1,7 +1,9 @@import pytestfrom deepteam.vulnerabilities import ToolOrchestrationAbuse-from deepteam.vulnerabilities.tool_orchestration_abuse import ToolOrchestrationAbuseType+from deepteam.vulnerabilities.tool_orchestration_abuse import (+ ToolOrchestrationAbuseType,+)from deepteam.test_case import RTTestCase@@ -15,33 +17,43 @@ class TestToolOrchestrationAbuse:"cross_tool_state_leakage",]tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in tool_orchestration_abuse.types+ ) == sorted(types)def test_tool_orchestration_abuse_all_types_default(self):tool_orchestration_abuse = ToolOrchestrationAbuse()- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(- type.value for type in ToolOrchestrationAbuseType- )+ assert sorted(+ type.value for type in tool_orchestration_abuse.types+ ) == sorted(type.value for type in ToolOrchestrationAbuseType)def test_tool_orchestration_abuse_recursive_tool_calls(self):types = ["recursive_tool_calls"]tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in tool_orchestration_abuse.types+ ) == sorted(types)def test_tool_orchestration_abuse_unsafe_tool_composition(self):types = ["unsafe_tool_composition"]tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in tool_orchestration_abuse.types+ ) == sorted(types)def test_tool_orchestration_abuse_tool_budget_exhaustion(self):types = ["tool_budget_exhaustion"]tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in tool_orchestration_abuse.types+ ) == sorted(types)def test_tool_orchestration_abuse_cross_tool_state_leakage(self):types = ["cross_tool_state_leakage"]tool_orchestration_abuse = ToolOrchestrationAbuse(types=types)- assert sorted(type.value for type in tool_orchestration_abuse.types) == sorted(types)+ assert sorted(+ type.value for type in tool_orchestration_abuse.types+ ) == sorted(types)def test_tool_orchestration_abuse_all_types_invalid(self):types = [@@ -55,16 +67,21 @@ class TestToolOrchestrationAbuse:ToolOrchestrationAbuse(types=types)def test_simulate_attacks_returns_expected_cases(self):- tool_orchestration_abuse = ToolOrchestrationAbuse(types=["recursive_tool_calls"])+ tool_orchestration_abuse = ToolOrchestrationAbuse(+ types=["recursive_tool_calls"]+ )test_cases = tool_orchestration_abuse.simulate_attacks(attacks_per_vulnerability_type=2)assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Tool Orchestration Abuse" for tc in test_cases)assert all(- tc.vulnerability_type == ToolOrchestrationAbuseType.RECURSIVE_TOOL_CALLS+ tc.vulnerability == "Tool Orchestration Abuse" for tc in test_cases+ )+ assert all(+ tc.vulnerability_type+ == ToolOrchestrationAbuseType.RECURSIVE_TOOL_CALLSfor tc in test_cases)@@ -81,13 +98,21 @@ class TestToolOrchestrationAbuse:)assert tool_orchestration_abuse.is_vulnerable() is not None- assert tool_orchestration_abuse.simulated_attacks is not None and isinstance(- tool_orchestration_abuse.simulated_attacks, dict+ assert (+ tool_orchestration_abuse.simulated_attacks is not None+ and isinstance(tool_orchestration_abuse.simulated_attacks, dict)+ )+ assert tool_orchestration_abuse.res is not None and isinstance(+ tool_orchestration_abuse.res, dict)- assert tool_orchestration_abuse.res is not None and isinstance(tool_orchestration_abuse.res, dict)assert ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION in results- assert len(results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION]) == 1- test_case = results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION][0]+ assert (+ len(results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION])+ == 1+ )+ test_case = results[ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION][+ 0+ ]# The score and reason will depend on the real metric evaluationassert hasattr(test_case, "score")assert hasattr(test_case, "reason")@@ -98,14 +123,18 @@ class TestToolOrchestrationAbuse:tool_orchestration_abuse = ToolOrchestrationAbuse(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = tool_orchestration_abuse._get_metric(ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION)+ metric = tool_orchestration_abuse._get_metric(+ ToolOrchestrationAbuseType.UNSAFE_TOOL_COMPOSITION+ )assert isinstance(metric, ToolOrchestrationMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- tool_orchestration_abuse = ToolOrchestrationAbuse(types=["cross_tool_state_leakage"], async_mode=True)+ tool_orchestration_abuse = ToolOrchestrationAbuse(+ types=["cross_tool_state_leakage"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -115,12 +144,20 @@ class TestToolOrchestrationAbuse:)assert tool_orchestration_abuse.is_vulnerable() is not None- assert tool_orchestration_abuse.simulated_attacks is not None and isinstance(- tool_orchestration_abuse.simulated_attacks, dict+ assert (+ tool_orchestration_abuse.simulated_attacks is not None+ and isinstance(tool_orchestration_abuse.simulated_attacks, dict)+ )+ assert tool_orchestration_abuse.res is not None and isinstance(+ tool_orchestration_abuse.res, dict)- assert tool_orchestration_abuse.res is not None and isinstance(tool_orchestration_abuse.res, dict)assert ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE in results- assert len(results[ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE]) == 1- test_case = results[ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE][0]+ assert (+ len(results[ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE])+ == 1+ )+ test_case = results[+ ToolOrchestrationAbuseType.CROSS_TOOL_STATE_LEAKAGE+ ][0]assert hasattr(test_case, "score")assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py b/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.pyindex bb78f45b..ed4a5b01 100644--- a/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py+++ b/tests/test_core/test_vulnerabilities/test_unexpected_code_execution.py@@ -1,7 +1,9 @@import pytestfrom deepteam.vulnerabilities import UnexpectedCodeExecution-from deepteam.vulnerabilities.unexpected_code_execution import UnexpectedCodeExecutionType+from deepteam.vulnerabilities.unexpected_code_execution import (+ UnexpectedCodeExecutionType,+)from deepteam.test_case import RTTestCase@@ -14,28 +16,36 @@ class TestUnexpectedCodeExecution:"eval_usage",]unexpected_code_execution = UnexpectedCodeExecution(types=types)- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)+ assert sorted(+ type.value for type in unexpected_code_execution.types+ ) == sorted(types)def test_unexpected_code_execution_all_types_default(self):unexpected_code_execution = UnexpectedCodeExecution()- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(- type.value for type in UnexpectedCodeExecutionType- )+ assert sorted(+ type.value for type in unexpected_code_execution.types+ ) == sorted(type.value for type in UnexpectedCodeExecutionType)def test_unexpected_code_execution_unauthorized_code_execution(self):types = ["unauthorized_code_execution"]unexpected_code_execution = UnexpectedCodeExecution(types=types)- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)+ assert sorted(+ type.value for type in unexpected_code_execution.types+ ) == sorted(types)def test_unexpected_code_execution_shell_command_execution(self):types = ["shell_command_execution"]unexpected_code_execution = UnexpectedCodeExecution(types=types)- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)+ assert sorted(+ type.value for type in unexpected_code_execution.types+ ) == sorted(types)def test_unexpected_code_execution_eval_usage(self):types = ["eval_usage"]unexpected_code_execution = UnexpectedCodeExecution(types=types)- assert sorted(type.value for type in unexpected_code_execution.types) == sorted(types)+ assert sorted(+ type.value for type in unexpected_code_execution.types+ ) == sorted(types)def test_unexpected_code_execution_all_types_invalid(self):types = [@@ -48,16 +58,21 @@ class TestUnexpectedCodeExecution:UnexpectedCodeExecution(types=types)def test_simulate_attacks_returns_expected_cases(self):- unexpected_code_execution = UnexpectedCodeExecution(types=["unauthorized_code_execution"])+ unexpected_code_execution = UnexpectedCodeExecution(+ types=["unauthorized_code_execution"]+ )test_cases = unexpected_code_execution.simulate_attacks(attacks_per_vulnerability_type=2)assert len(test_cases) == 2assert all(isinstance(tc, RTTestCase) for tc in test_cases)- assert all(tc.vulnerability == "Unexpected Code Execution" for tc in test_cases)assert all(- tc.vulnerability_type == UnexpectedCodeExecutionType.UNAUTHORIZED_CODE_EXECUTION+ tc.vulnerability == "Unexpected Code Execution" for tc in test_cases+ )+ assert all(+ tc.vulnerability_type+ == UnexpectedCodeExecutionType.UNAUTHORIZED_CODE_EXECUTIONfor tc in test_cases)@@ -74,13 +89,21 @@ class TestUnexpectedCodeExecution:)assert unexpected_code_execution.is_vulnerable() is not None- assert unexpected_code_execution.simulated_attacks is not None and isinstance(- unexpected_code_execution.simulated_attacks, dict+ assert (+ unexpected_code_execution.simulated_attacks is not None+ and isinstance(unexpected_code_execution.simulated_attacks, dict)+ )+ assert unexpected_code_execution.res is not None and isinstance(+ unexpected_code_execution.res, dict)- assert unexpected_code_execution.res is not None and isinstance(unexpected_code_execution.res, dict)assert UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION in results- assert len(results[UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION]) == 1- test_case = results[UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION][0]+ assert (+ len(results[UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION])+ == 1+ )+ test_case = results[+ UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION+ ][0]# The score and reason will depend on the real metric evaluationassert hasattr(test_case, "score")assert hasattr(test_case, "reason")@@ -91,14 +114,18 @@ class TestUnexpectedCodeExecution:unexpected_code_execution = UnexpectedCodeExecution(async_mode=True, verbose_mode=True, evaluation_model="gpt-4o")- metric = unexpected_code_execution._get_metric(UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION)+ metric = unexpected_code_execution._get_metric(+ UnexpectedCodeExecutionType.SHELL_COMMAND_EXECUTION+ )assert isinstance(metric, UnexpectedCodeExecutionMetric)assert metric.async_mode is Trueassert metric.verbose_mode is True@pytest.mark.asyncioasync def test_a_assess_returns_async_results(self):- unexpected_code_execution = UnexpectedCodeExecution(types=["eval_usage"], async_mode=True)+ unexpected_code_execution = UnexpectedCodeExecution(+ types=["eval_usage"], async_mode=True+ )async def dummy_model_callback(prompt):return prompt@@ -108,10 +135,13 @@ class TestUnexpectedCodeExecution:)assert unexpected_code_execution.is_vulnerable() is not None- assert unexpected_code_execution.simulated_attacks is not None and isinstance(- unexpected_code_execution.simulated_attacks, dict+ assert (+ unexpected_code_execution.simulated_attacks is not None+ and isinstance(unexpected_code_execution.simulated_attacks, dict)+ )+ assert unexpected_code_execution.res is not None and isinstance(+ unexpected_code_execution.res, dict)- assert unexpected_code_execution.res is not None and isinstance(unexpected_code_execution.res, dict)assert UnexpectedCodeExecutionType.EVAL_USAGE in resultsassert len(results[UnexpectedCodeExecutionType.EVAL_USAGE]) == 1test_case = results[UnexpectedCodeExecutionType.EVAL_USAGE][0]diff --git a/tests/test_frameworks/test_owasp_top_10_asi.py b/tests/test_frameworks/test_owasp_top_10_asi.pyindex 64e2d2f4..df11f492 100644--- a/tests/test_frameworks/test_owasp_top_10_asi.py+++ b/tests/test_frameworks/test_owasp_top_10_asi.py@@ -3,7 +3,9 @@ from deepteam.frameworks import OWASP_ASI_2026from deepteam.red_teamer.risk_assessment import RiskAssessmentfrom deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.attacks import BaseAttack-from deepteam.frameworks.owasp_top_10_agentic.risk_categories import OWASP_ASI_CATEGORIES+from deepteam.frameworks.owasp_top_10_agentic.risk_categories import (+ OWASP_ASI_CATEGORIES,+)from deepteam.frameworks.risk_category import RiskCategoryfrom deepteam import red_team