DeepTeam is a framework to red team LLMs and AI agents.
added new tests for new vulnerabilities
A-Vamshib16a8ba5cb04
diff --git a/tests/test_core/test_vulnerabilities/test_cross_context_retrieval.py b/tests/test_core/test_vulnerabilities/test_cross_context_retrieval.pynew file mode 100644index 00000000..fff69b09--- /dev/null+++ b/tests/test_core/test_vulnerabilities/test_cross_context_retrieval.py@@ -0,0 +1,106 @@+import pytest++from deepteam.vulnerabilities import CrossContextRetrieval+from deepteam.vulnerabilities.cross_context_retrieval import CrossContextRetrievalType+from deepteam.test_case import RTTestCase+++class TestCrossContextRetrieval:++ def test_cross_context_retrieval_all_types(self):+ types = ["tenant", "user", "role"]+ cross_context_retrieval = CrossContextRetrieval(types=types)+ assert sorted(type.value for type in cross_context_retrieval.types) == sorted(types)++ def test_cross_context_retrieval_all_types_default(self):+ cross_context_retrieval = CrossContextRetrieval()+ assert sorted(type.value for type in cross_context_retrieval.types) == sorted(+ type.value for type in CrossContextRetrievalType+ )++ def test_cross_context_retrieval_tenant(self):+ types = ["tenant"]+ cross_context_retrieval = CrossContextRetrieval(types=types)+ assert sorted(type.value for type in cross_context_retrieval.types) == sorted(types)++ def test_cross_context_retrieval_user(self):+ types = ["user"]+ cross_context_retrieval = CrossContextRetrieval(types=types)+ assert sorted(type.value for type in cross_context_retrieval.types) == sorted(types)++ def test_cross_context_retrieval_role(self):+ types = ["role"]+ cross_context_retrieval = CrossContextRetrieval(types=types)+ assert sorted(type.value for type in cross_context_retrieval.types) == sorted(types)++ def test_cross_context_retrieval_all_types_invalid(self):+ types = ["tenant", "user", "role", "invalid"]+ with pytest.raises(ValueError):+ CrossContextRetrieval(types=types)++ def test_simulate_attacks_returns_expected_cases(self):+ cross_context_retrieval = CrossContextRetrieval(types=["tenant"])+ test_cases = cross_context_retrieval.simulate_attacks(attacks_per_vulnerability_type=2)++ assert len(test_cases) == 2+ assert all(isinstance(tc, RTTestCase) for tc in test_cases)+ assert all(tc.vulnerability == "Cross-Context Retrieval" for tc in test_cases)+ assert all(+ tc.vulnerability_type == CrossContextRetrievalType.TENANT for tc in test_cases+ )++ def test_assess_returns_results(self):+ cross_context_retrieval = CrossContextRetrieval(types=["user"], async_mode=False)++ def dummy_model_callback(prompt):+ # Provide a simple pass-through or minimal callback if required by your real env+ return prompt++ results = cross_context_retrieval.assess(+ model_callback=dummy_model_callback,+ )++ assert cross_context_retrieval.is_vulnerable() is not None+ assert cross_context_retrieval.simulated_attacks is not None and isinstance(+ cross_context_retrieval.simulated_attacks, dict+ )+ assert cross_context_retrieval.res is not None and isinstance(cross_context_retrieval.res, dict)+ assert CrossContextRetrievalType.USER in results+ assert len(results[CrossContextRetrievalType.USER]) == 1+ test_case = results[CrossContextRetrievalType.USER][0]+ # The score and reason will depend on the real metric evaluation+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")++ def test_get_metric_returns_cross_context_retrieval_metric(self):+ from deepteam.metrics import CrossContextRetrievalMetric++ cross_context_retrieval = CrossContextRetrieval(+ async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"+ )+ metric = cross_context_retrieval._get_metric(CrossContextRetrievalType.ROLE)+ assert isinstance(metric, CrossContextRetrievalMetric)+ assert metric.async_mode is True+ assert metric.verbose_mode is True++ @pytest.mark.asyncio+ async def test_a_assess_returns_async_results(self):+ cross_context_retrieval = CrossContextRetrieval(types=["user"], async_mode=True)++ async def dummy_model_callback(prompt):+ return prompt++ results = await cross_context_retrieval.a_assess(+ model_callback=dummy_model_callback,+ )++ assert cross_context_retrieval.is_vulnerable() is not None+ assert cross_context_retrieval.simulated_attacks is not None and isinstance(+ cross_context_retrieval.simulated_attacks, dict+ )+ assert cross_context_retrieval.res is not None and isinstance(cross_context_retrieval.res, dict)+ assert CrossContextRetrievalType.USER in results+ assert len(results[CrossContextRetrievalType.USER]) == 1+ test_case = results[CrossContextRetrievalType.USER][0]+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_exploit_tool_agent.py b/tests/test_core/test_vulnerabilities/test_exploit_tool_agent.pynew file mode 100644index 00000000..be37b39d--- /dev/null+++ b/tests/test_core/test_vulnerabilities/test_exploit_tool_agent.py@@ -0,0 +1,106 @@+import pytest++from deepteam.vulnerabilities import ExploitToolAgent+from deepteam.vulnerabilities.exploit_tool_agent import ExploitToolAgentType+from deepteam.test_case import RTTestCase+++class TestExploitToolAgent:++ def test_exploit_tool_agent_all_types(self):+ types = ["privilege_escalation", "financial_manipulation", "data_destruction"]+ exploit_tool_agent = ExploitToolAgent(types=types)+ assert sorted(type.value for type in exploit_tool_agent.types) == sorted(types)++ def test_exploit_tool_agent_all_types_default(self):+ exploit_tool_agent = ExploitToolAgent()+ assert sorted(type.value for type in exploit_tool_agent.types) == sorted(+ type.value for type in ExploitToolAgentType+ )++ def test_exploit_tool_agent_privilege_escalation(self):+ types = ["privilege_escalation"]+ exploit_tool_agent = ExploitToolAgent(types=types)+ assert sorted(type.value for type in exploit_tool_agent.types) == sorted(types)++ def test_exploit_tool_agent_financial_manipulation(self):+ types = ["financial_manipulation"]+ exploit_tool_agent = ExploitToolAgent(types=types)+ assert sorted(type.value for type in exploit_tool_agent.types) == sorted(types)++ def test_exploit_tool_agent_data_destruction(self):+ types = ["data_destruction"]+ exploit_tool_agent = ExploitToolAgent(types=types)+ assert sorted(type.value for type in exploit_tool_agent.types) == sorted(types)++ def test_exploit_tool_agent_all_types_invalid(self):+ types = ["privilege_escalation", "financial_manipulation", "data_destruction", "invalid"]+ with pytest.raises(ValueError):+ ExploitToolAgent(types=types)++ def test_simulate_attacks_returns_expected_cases(self):+ exploit_tool_agent = ExploitToolAgent(types=["privilege_escalation"])+ test_cases = exploit_tool_agent.simulate_attacks(attacks_per_vulnerability_type=2)++ assert len(test_cases) == 2+ assert all(isinstance(tc, RTTestCase) for tc in test_cases)+ assert all(tc.vulnerability == "Exploit Tool Agent" for tc in test_cases)+ assert all(+ tc.vulnerability_type == ExploitToolAgentType.PRIVILEGE_ESCALATION for tc in test_cases+ )++ def test_assess_returns_results(self):+ exploit_tool_agent = ExploitToolAgent(types=["data_destruction"], async_mode=False)++ def dummy_model_callback(prompt):+ # Provide a simple pass-through or minimal callback if required by your real env+ return prompt++ results = exploit_tool_agent.assess(+ model_callback=dummy_model_callback,+ )++ assert exploit_tool_agent.is_vulnerable() is not None+ assert exploit_tool_agent.simulated_attacks is not None and isinstance(+ exploit_tool_agent.simulated_attacks, dict+ )+ assert exploit_tool_agent.res is not None and isinstance(exploit_tool_agent.res, dict)+ assert ExploitToolAgentType.DATA_DESTRUCTION in results+ assert len(results[ExploitToolAgentType.DATA_DESTRUCTION]) == 1+ test_case = results[ExploitToolAgentType.DATA_DESTRUCTION][0]+ # The score and reason will depend on the real metric evaluation+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")++ def test_get_metric_returns_cross_context_retrieval_metric(self):+ from deepteam.metrics import ExploitToolAgentMetric++ exploit_tool_agent = ExploitToolAgent(+ async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"+ )+ metric = exploit_tool_agent._get_metric(ExploitToolAgentType.FINANCIAL_MANIPULATION)+ assert isinstance(metric, ExploitToolAgentMetric)+ assert metric.async_mode is True+ assert metric.verbose_mode is True++ @pytest.mark.asyncio+ async def test_a_assess_returns_async_results(self):+ exploit_tool_agent = ExploitToolAgent(types=["financial_manipulation"], async_mode=True)++ async def dummy_model_callback(prompt):+ return prompt++ results = await exploit_tool_agent.a_assess(+ model_callback=dummy_model_callback,+ )++ assert exploit_tool_agent.is_vulnerable() is not None+ assert exploit_tool_agent.simulated_attacks is not None and isinstance(+ exploit_tool_agent.simulated_attacks, dict+ )+ assert exploit_tool_agent.res is not None and isinstance(exploit_tool_agent.res, dict)+ assert ExploitToolAgentType.FINANCIAL_MANIPULATION in results+ assert len(results[ExploitToolAgentType.FINANCIAL_MANIPULATION]) == 1+ test_case = results[ExploitToolAgentType.FINANCIAL_MANIPULATION][0]+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_external_system_abuse.py b/tests/test_core/test_vulnerabilities/test_external_system_abuse.pynew file mode 100644index 00000000..0bb849c0--- /dev/null+++ b/tests/test_core/test_vulnerabilities/test_external_system_abuse.py@@ -0,0 +1,106 @@+import pytest++from deepteam.vulnerabilities import ExternalSystemAbuse+from deepteam.vulnerabilities.external_system_abuse import ExternalSystemAbuseType+from deepteam.test_case import RTTestCase+++class TestExternalSystemAbuse:++ def test_external_system_abuse_all_types(self):+ types = ["data_exfiltration", "communications_spam", "internal_spoofing"]+ external_system_abuse = ExternalSystemAbuse(types=types)+ assert sorted(type.value for type in external_system_abuse.types) == sorted(types)++ def test_external_system_abuse_all_types_default(self):+ external_system_abuse = ExternalSystemAbuse()+ assert sorted(type.value for type in external_system_abuse.types) == sorted(+ type.value for type in ExternalSystemAbuseType+ )++ def test_external_system_abuse_data_exfiltration(self):+ types = ["data_exfiltration"]+ external_system_abuse = ExternalSystemAbuse(types=types)+ assert sorted(type.value for type in external_system_abuse.types) == sorted(types)++ def test_external_system_abuse_communications_spam(self):+ types = ["communications_spam"]+ external_system_abuse = ExternalSystemAbuse(types=types)+ assert sorted(type.value for type in external_system_abuse.types) == sorted(types)++ def test_external_system_abuse_internal_spoofing(self):+ types = ["internal_spoofing"]+ external_system_abuse = ExternalSystemAbuse(types=types)+ assert sorted(type.value for type in external_system_abuse.types) == sorted(types)++ def test_external_system_abuse_all_types_invalid(self):+ types = ["data_exfiltration", "communications_spam", "internal_spoofing", "invalid"]+ with pytest.raises(ValueError):+ ExternalSystemAbuse(types=types)++ def test_simulate_attacks_returns_expected_cases(self):+ external_system_abuse = ExternalSystemAbuse(types=["data_exfiltration"])+ test_cases = external_system_abuse.simulate_attacks(attacks_per_vulnerability_type=2)++ assert len(test_cases) == 2+ assert all(isinstance(tc, RTTestCase) for tc in test_cases)+ assert all(tc.vulnerability == "External System Abuse" for tc in test_cases)+ assert all(+ tc.vulnerability_type == ExternalSystemAbuseType.DATA_EXFILTRATION for tc in test_cases+ )++ def test_assess_returns_results(self):+ external_system_abuse = ExternalSystemAbuse(types=["communications_spam"], async_mode=False)++ def dummy_model_callback(prompt):+ # Provide a simple pass-through or minimal callback if required by your real env+ return prompt++ results = external_system_abuse.assess(+ model_callback=dummy_model_callback,+ )++ assert external_system_abuse.is_vulnerable() is not None+ assert external_system_abuse.simulated_attacks is not None and isinstance(+ external_system_abuse.simulated_attacks, dict+ )+ assert external_system_abuse.res is not None and isinstance(external_system_abuse.res, dict)+ assert ExternalSystemAbuseType.COMMUNICATIONS_SPAM in results+ assert len(results[ExternalSystemAbuseType.COMMUNICATIONS_SPAM]) == 1+ test_case = results[ExternalSystemAbuseType.COMMUNICATIONS_SPAM][0]+ # The score and reason will depend on the real metric evaluation+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")++ def test_get_metric_returns_cross_context_retrieval_metric(self):+ from deepteam.metrics import ExternalSystemAbuseMetric++ external_system_abuse = ExternalSystemAbuse(+ async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"+ )+ metric = external_system_abuse._get_metric(ExternalSystemAbuseType.INTERNAL_SPOOFING)+ assert isinstance(metric, ExternalSystemAbuseMetric)+ assert metric.async_mode is True+ assert metric.verbose_mode is True++ @pytest.mark.asyncio+ async def test_a_assess_returns_async_results(self):+ external_system_abuse = ExternalSystemAbuse(types=["internal_spoofing"], async_mode=True)++ async def dummy_model_callback(prompt):+ return prompt++ results = await external_system_abuse.a_assess(+ model_callback=dummy_model_callback,+ )++ assert external_system_abuse.is_vulnerable() is not None+ assert external_system_abuse.simulated_attacks is not None and isinstance(+ external_system_abuse.simulated_attacks, dict+ )+ assert external_system_abuse.res is not None and isinstance(external_system_abuse.res, dict)+ assert ExternalSystemAbuseType.INTERNAL_SPOOFING in results+ assert len(results[ExternalSystemAbuseType.INTERNAL_SPOOFING]) == 1+ test_case = results[ExternalSystemAbuseType.INTERNAL_SPOOFING][0]+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_system_reconnaissance.py b/tests/test_core/test_vulnerabilities/test_system_reconnaissance.pynew file mode 100644index 00000000..fe53b231--- /dev/null+++ b/tests/test_core/test_vulnerabilities/test_system_reconnaissance.py@@ -0,0 +1,106 @@+import pytest++from deepteam.vulnerabilities import SystemReconnaissance+from deepteam.vulnerabilities.system_reconnaissance import SystemReconnaissanceType+from deepteam.test_case import RTTestCase+++class TestSystemReconnaissance:++ def test_system_reconnaissance_all_types(self):+ types = ["file_metadata", "database_schema", "retrieval_config"]+ system_reconnaissance = SystemReconnaissance(types=types)+ assert sorted(type.value for type in system_reconnaissance.types) == sorted(types)++ def test_system_reconnaissance_all_types_default(self):+ system_reconnaissance = SystemReconnaissance()+ assert sorted(type.value for type in system_reconnaissance.types) == sorted(+ type.value for type in SystemReconnaissanceType+ )++ def test_system_reconnaissance_file_metadata(self):+ types = ["file_metadata"]+ system_reconnaissance = SystemReconnaissance(types=types)+ assert sorted(type.value for type in system_reconnaissance.types) == sorted(types)++ def test_system_reconnaissance_database_schema(self):+ types = ["database_schema"]+ system_reconnaissance = SystemReconnaissance(types=types)+ assert sorted(type.value for type in system_reconnaissance.types) == sorted(types)++ def test_system_reconnaissance_retrieval_config(self):+ types = ["retrieval_config"]+ system_reconnaissance = SystemReconnaissance(types=types)+ assert sorted(type.value for type in system_reconnaissance.types) == sorted(types)++ def test_system_reconnaissance_all_types_invalid(self):+ types = ["file_metadata", "database_schema", "retrieval_config", "invalid"]+ with pytest.raises(ValueError):+ SystemReconnaissance(types=types)++ def test_simulate_attacks_returns_expected_cases(self):+ system_reconnaissance = SystemReconnaissance(types=["file_metadata"])+ test_cases = system_reconnaissance.simulate_attacks(attacks_per_vulnerability_type=2)++ assert len(test_cases) == 2+ assert all(isinstance(tc, RTTestCase) for tc in test_cases)+ assert all(tc.vulnerability == "System Reconnaissance" for tc in test_cases)+ assert all(+ tc.vulnerability_type == SystemReconnaissanceType.FILE_METADATA for tc in test_cases+ )++ def test_assess_returns_results(self):+ system_reconnaissance = SystemReconnaissance(types=["database_schema"], async_mode=False)++ def dummy_model_callback(prompt):+ # Provide a simple pass-through or minimal callback if required by your real env+ return prompt++ results = system_reconnaissance.assess(+ model_callback=dummy_model_callback,+ )++ assert system_reconnaissance.is_vulnerable() is not None+ assert system_reconnaissance.simulated_attacks is not None and isinstance(+ system_reconnaissance.simulated_attacks, dict+ )+ assert system_reconnaissance.res is not None and isinstance(system_reconnaissance.res, dict)+ assert SystemReconnaissanceType.DATABASE_SCHEMA in results+ assert len(results[SystemReconnaissanceType.DATABASE_SCHEMA]) == 1+ test_case = results[SystemReconnaissanceType.DATABASE_SCHEMA][0]+ # The score and reason will depend on the real metric evaluation+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")++ def test_get_metric_returns_cross_context_retrieval_metric(self):+ from deepteam.metrics import SystemReconnaissanceMetric++ system_reconnaissance = SystemReconnaissance(+ async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"+ )+ metric = system_reconnaissance._get_metric(SystemReconnaissanceType.FILE_METADATA)+ assert isinstance(metric, SystemReconnaissanceMetric)+ assert metric.async_mode is True+ assert metric.verbose_mode is True++ @pytest.mark.asyncio+ async def test_a_assess_returns_async_results(self):+ system_reconnaissance = SystemReconnaissance(types=["retrieval_config"], async_mode=True)++ async def dummy_model_callback(prompt):+ return prompt++ results = await system_reconnaissance.a_assess(+ model_callback=dummy_model_callback,+ )++ assert system_reconnaissance.is_vulnerable() is not None+ assert system_reconnaissance.simulated_attacks is not None and isinstance(+ system_reconnaissance.simulated_attacks, dict+ )+ assert system_reconnaissance.res is not None and isinstance(system_reconnaissance.res, dict)+ assert SystemReconnaissanceType.RETRIEVAL_CONFIG in results+ assert len(results[SystemReconnaissanceType.RETRIEVAL_CONFIG]) == 1+ test_case = results[SystemReconnaissanceType.RETRIEVAL_CONFIG][0]+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")