DeepTeam is a framework to red team LLMs and AI agents.
added tests for ethics and fairness vulnerabilities
A-Vamshi8f3001075499
diff --git a/tests/test_core/test_vulnerabilities/test_all.py b/tests/test_core/test_vulnerabilities/test_all.pyindex 95f16c08..058174e6 100644--- a/tests/test_core/test_vulnerabilities/test_all.py+++ b/tests/test_core/test_vulnerabilities/test_all.py@@ -18,6 +18,8 @@ def test_red_teaming():vulnerabilities = [Bias(),+ Ethics(),+ Fairness(),Toxicity(),Misinformation(),IllegalActivity(),diff --git a/tests/test_core/test_vulnerabilities/test_ethics.py b/tests/test_core/test_vulnerabilities/test_ethics.pynew file mode 100644index 00000000..19101fb2--- /dev/null+++ b/tests/test_core/test_vulnerabilities/test_ethics.py@@ -0,0 +1,106 @@+import pytest++from deepteam.vulnerabilities import Ethics+from deepteam.vulnerabilities.ethics import EthicsType+from deepteam.test_case import RTTestCase+++class TestEthics:++ def test_ethics_all_types(self):+ types = ["harm_prevention", "moral_integrity", "responsible_transparency"]+ ethics = Ethics(types=types)+ assert sorted(type.value for type in ethics.types) == sorted(types)++ def test_ethics_all_types_default(self):+ ethics = Ethics()+ assert sorted(type.value for type in ethics.types) == sorted(+ type.value for type in EthicsType+ )++ def test_ethics_harm_prevention(self):+ types = ["harm_prevention"]+ ethics = Ethics(types=types)+ assert sorted(type.value for type in ethics.types) == sorted(types)++ def test_ethics_moral_integrity(self):+ types = ["moral_integrity"]+ ethics = Ethics(types=types)+ assert sorted(type.value for type in ethics.types) == sorted(types)++ def test_ethics_responsible_transparency(self):+ types = ["responsible_transparency"]+ ethics = Ethics(types=types)+ assert sorted(type.value for type in ethics.types) == sorted(types)++ def test_ethics_all_types_invalid(self):+ types = ["harm_prevention", "moral_integrity", "responsible_transparency", "invalid"]+ with pytest.raises(ValueError):+ Ethics(types=types)++ def test_simulate_attacks_returns_expected_cases(self):+ ethics = Ethics(types=["harm_prevention"])+ test_cases = ethics.simulate_attacks(attacks_per_vulnerability_type=2)++ assert len(test_cases) == 2+ assert all(isinstance(tc, RTTestCase) for tc in test_cases)+ assert all(tc.vulnerability == "Ethics" for tc in test_cases)+ assert all(+ tc.vulnerability_type == EthicsType.HARM_PREVENTION for tc in test_cases+ )++ def test_assess_returns_results(self):+ ethics = Ethics(types=["moral_integrity"], async_mode=False)++ def dummy_model_callback(prompt):+ # Provide a simple pass-through or minimal callback if required by your real env+ return prompt++ results = ethics.assess(+ model_callback=dummy_model_callback,+ )++ assert ethics.is_vulnerable() is not None+ assert ethics.simulated_attacks is not None and isinstance(+ ethics.simulated_attacks, dict+ )+ assert ethics.res is not None and isinstance(ethics.res, dict)+ assert EthicsType.MORAL_INTEGRITY in results+ assert len(results[EthicsType.MORAL_INTEGRITY]) == 1+ test_case = results[EthicsType.MORAL_INTEGRITY][0]+ # The score and reason will depend on the real metric evaluation+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")++ def test_get_metric_returns_ethics_metric(self):+ from deepteam.metrics import EthicsMetric++ ethics = Ethics(+ async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"+ )+ metric = ethics._get_metric(EthicsType.HARM_PREVENTION)+ assert isinstance(metric, EthicsMetric)+ assert metric.async_mode is True+ assert metric.verbose_mode is True++ @pytest.mark.asyncio+ async def test_a_assess_returns_async_results(self):+ ethics = Ethics(types=["moral_integrity"], async_mode=True)++ async def dummy_model_callback(prompt):+ return prompt++ results = await ethics.a_assess(+ model_callback=dummy_model_callback,+ )++ assert ethics.is_vulnerable() is not None+ assert ethics.simulated_attacks is not None and isinstance(+ ethics.simulated_attacks, dict+ )+ assert ethics.res is not None and isinstance(ethics.res, dict)+ assert EthicsType.MORAL_INTEGRITY in results+ assert len(results[EthicsType.MORAL_INTEGRITY]) == 1+ test_case = results[EthicsType.MORAL_INTEGRITY][0]+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")diff --git a/tests/test_core/test_vulnerabilities/test_fairness.py b/tests/test_core/test_vulnerabilities/test_fairness.pynew file mode 100644index 00000000..7b5b001f--- /dev/null+++ b/tests/test_core/test_vulnerabilities/test_fairness.py@@ -0,0 +1,106 @@+import pytest++from deepteam.vulnerabilities import Fairness+from deepteam.vulnerabilities.fairness import FairnessType+from deepteam.test_case import RTTestCase+++class TestFairness:++ def test_fairness_all_types(self):+ types = ["equality_consistency", "procedural_opportunity", "temporal_outcome"]+ fairness = Fairness(types=types)+ assert sorted(type.value for type in fairness.types) == sorted(types)++ def test_fairness_all_types_default(self):+ fairness = Fairness()+ assert sorted(type.value for type in fairness.types) == sorted(+ type.value for type in FairnessType+ )++ def test_fairness_equality_consistency(self):+ types = ["equality_consistency"]+ fairness = Fairness(types=types)+ assert sorted(type.value for type in fairness.types) == sorted(types)++ def test_fairness_procedural_opportunity(self):+ types = ["procedural_opportunity"]+ fairness = Fairness(types=types)+ assert sorted(type.value for type in fairness.types) == sorted(types)++ def test_fairness_temporal_outcome(self):+ types = ["temporal_outcome"]+ fairness = Fairness(types=types)+ assert sorted(type.value for type in fairness.types) == sorted(types)++ def test_fairness_all_types_invalid(self):+ types = ["equality_consistency", "procedural_opportunity", "temporal_outcome", "invalid"]+ with pytest.raises(ValueError):+ Fairness(types=types)++ def test_simulate_attacks_returns_expected_cases(self):+ fairness = Fairness(types=["equality_consistency"])+ test_cases = fairness.simulate_attacks(attacks_per_vulnerability_type=2)++ assert len(test_cases) == 2+ assert all(isinstance(tc, RTTestCase) for tc in test_cases)+ assert all(tc.vulnerability == "Fairness" for tc in test_cases)+ assert all(+ tc.vulnerability_type == FairnessType.EQUALITY_CONSISTENCY for tc in test_cases+ )++ def test_assess_returns_results(self):+ fairness = Fairness(types=["procedural_opportunity"], async_mode=False)++ def dummy_model_callback(prompt):+ # Provide a simple pass-through or minimal callback if required by your real env+ return prompt++ results = fairness.assess(+ model_callback=dummy_model_callback,+ )++ assert fairness.is_vulnerable() is not None+ assert fairness.simulated_attacks is not None and isinstance(+ fairness.simulated_attacks, dict+ )+ assert fairness.res is not None and isinstance(fairness.res, dict)+ assert FairnessType.PROCEDURAL_OPPORTUNITY in results+ assert len(results[FairnessType.PROCEDURAL_OPPORTUNITY]) == 1+ test_case = results[FairnessType.PROCEDURAL_OPPORTUNITY][0]+ # The score and reason will depend on the real metric evaluation+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")++ def test_get_metric_returns_fairness_metric(self):+ from deepteam.metrics import FairnessMetric++ fairness = Fairness(+ async_mode=True, verbose_mode=True, evaluation_model="gpt-4o"+ )+ metric = fairness._get_metric(FairnessType.PROCEDURAL_OPPORTUNITY)+ assert isinstance(metric, FairnessMetric)+ assert metric.async_mode is True+ assert metric.verbose_mode is True++ @pytest.mark.asyncio+ async def test_a_assess_returns_async_results(self):+ fairness = Fairness(types=["equality_consistency"], async_mode=True)++ async def dummy_model_callback(prompt):+ return prompt++ results = await fairness.a_assess(+ model_callback=dummy_model_callback,+ )++ assert fairness.is_vulnerable() is not None+ assert fairness.simulated_attacks is not None and isinstance(+ fairness.simulated_attacks, dict+ )+ assert fairness.res is not None and isinstance(fairness.res, dict)+ assert FairnessType.EQUALITY_CONSISTENCY in results+ assert len(results[FairnessType.EQUALITY_CONSISTENCY]) == 1+ test_case = results[FairnessType.EQUALITY_CONSISTENCY][0]+ assert hasattr(test_case, "score")+ assert hasattr(test_case, "reason")