DeepTeam is a framework to red team LLMs and AI agents.
Update tests for sample rate validation and deterministic behavior verification
sid-murali1f2903f024da
diff --git a/deepteam/guardrails/guardrails.py b/deepteam/guardrails/guardrails.pyindex 6951aebd..63fff38d 100644--- a/deepteam/guardrails/guardrails.py+++ b/deepteam/guardrails/guardrails.py@@ -44,7 +44,7 @@ class Guardrails:self.sample_rate = sample_rateself.evaluation_model = evaluation_model- self._request_count = 0 # Deterministic counter+ self._request_count = 0# Update all guards to use the specified evaluation modelself.input_guards = self._update_guards_model(input_guards, evaluation_model)diff --git a/deepteam/guardrails/guards/cybersecurity_guard.py b/deepteam/guardrails/guards/cybersecurity_guard.pyindex dccb692a..fc08027a 100644--- a/deepteam/guardrails/guards/cybersecurity_guard.py+++ b/deepteam/guardrails/guards/cybersecurity_guard.py@@ -11,6 +11,7 @@ class CybersecurityGuard(BaseGuard):purpose: Optional[str] = None,categories: List[CyberattackCategory] = None,guard_type: GuardType = GuardType.INPUT,+ evaluation_model: Optional[str] = None,):super().__init__(evaluation_model=evaluation_model)self.guard_type = guard_typediff --git a/tests/test_input_output_guards.py b/tests/test_input_output_guards.pyindex d8bcfa50..72957808 100644--- a/tests/test_input_output_guards.py+++ b/tests/test_input_output_guards.py@@ -46,63 +46,93 @@ def test_evaluation_model():print("β Custom evaluation model works")-def test_sample_rate():- """Test sample rate functionality"""+def test_sample_rate_negative():+ """Test sample rate -0.2 (should fail)"""from deepteam.guardrails import Guardrailsfrom deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard- # Test with 0% sample rate (should never run guards)+ print("π§ͺ Testing sample_rate = -0.2")guardrails = Guardrails(input_guards=[PromptInjectionGuard()],output_guards=[ToxicityGuard()],- sample_rate=0.0+ sample_rate=-0.2)+ print("β Sample rate -0.2 somehow worked")+++def test_sample_rate_valid_low():+ """Test sample rate 0.2 (should work)"""- result = guardrails.guard_input(input="Test input")- assert len(result.guard_results) == 0 # Should be empty due to 0% sampling- print("β Sample rate 0.0 works (no guards executed)")+ from deepteam.guardrails import Guardrails+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard- # Test with 100% sample rate (should always run guards)+ print("π§ͺ Testing sample_rate = 0.2")guardrails = Guardrails(input_guards=[PromptInjectionGuard()],output_guards=[ToxicityGuard()],- sample_rate=1.0+ sample_rate=0.2)- result = guardrails.guard_input(input="Test input")- assert len(result.guard_results) > 0 # Should have results due to 100% sampling- print("β Sample rate 1.0 works (guards executed)")+ # Test deterministic behavior - should process every 5th request+ processed = 0+ for i in range(10):+ result = guardrails.guard_input(f"test {i}")+ if len(result.guard_results) > 0:+ processed += 1++ print(f"β Sample rate 0.2 works - processed {processed}/10 requests")-def test_sample_rate_validation():- """Test sample rate validation"""+def test_sample_rate_zero():+ """Test sample rate 0.0 (should work)"""from deepteam.guardrails import Guardrailsfrom deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard- # Test invalid sample rates- try:- Guardrails(- input_guards=[PromptInjectionGuard()],- output_guards=[ToxicityGuard()],- sample_rate=1.5 # Invalid: > 1.0- )- assert False, "Should have raised ValueError"- except ValueError:- pass+ print("π§ͺ Testing sample_rate = 0.0")+ guardrails = Guardrails(+ input_guards=[PromptInjectionGuard()],+ output_guards=[ToxicityGuard()],+ sample_rate=0.0+ )- try:- Guardrails(- input_guards=[PromptInjectionGuard()],- output_guards=[ToxicityGuard()],- sample_rate=-0.1 # Invalid: < 0.0- )- assert False, "Should have raised ValueError"- except ValueError:- pass+ result = guardrails.guard_input("test")+ assert len(result.guard_results) == 0+ print("β Sample rate 0.0 works - no guards executed")+++def test_sample_rate_too_high():+ """Test sample rate 1.3 (should fail)"""- print("β Sample rate validation works")+ from deepteam.guardrails import Guardrails+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard++ print("π§ͺ Testing sample_rate = 1.3")+ guardrails = Guardrails(+ input_guards=[PromptInjectionGuard()],+ output_guards=[ToxicityGuard()],+ sample_rate=1.3+ )+ print("β Sample rate 1.3 somehow worked")+++def test_sample_rate_one():+ """Test sample rate 1 (should work)"""++ from deepteam.guardrails import Guardrails+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard++ print("π§ͺ Testing sample_rate = 1")+ guardrails = Guardrails(+ input_guards=[PromptInjectionGuard()],+ output_guards=[ToxicityGuard()],+ sample_rate=1+ )++ result = guardrails.guard_input("test")+ assert len(result.guard_results) > 0+ print("β Sample rate 1 works - guards executed")def test_error_handling():@@ -157,8 +187,9 @@ if __name__ == "__main__":test_basic_functionality()test_evaluation_model()- test_sample_rate()- test_sample_rate_validation()+ test_sample_rate_valid_low() # Should work+ test_sample_rate_zero() # Should work+ test_sample_rate_one()test_error_handling()test_attack_detection()diff --git a/tests/test_sample_rates.py b/tests/test_sample_rates.pynew file mode 100644index 00000000..33380dd5--- /dev/null+++ b/tests/test_sample_rates.py@@ -0,0 +1,58 @@+#!/usr/bin/env python3+"""+Test different sample rate values to see error messages.+"""++def test_sample_rate(value, description):+ """Test a specific sample rate value"""+ print(f"\nπ§ͺ Testing sample_rate = {value} ({description})")++ try:+ from deepteam.guardrails import Guardrails+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard++ guardrails = Guardrails(+ input_guards=[PromptInjectionGuard()],+ output_guards=[ToxicityGuard()],+ sample_rate=value+ )++ print(f"β Sample rate {value} worked!")++ # Quick test to see deterministic behavior+ if value == 0.2:+ processed = 0+ for i in range(10):+ result = guardrails.guard_input(f"test {i}")+ if len(result.guard_results) > 0:+ processed += 1+ print(f" β Processed {processed}/10 requests (expected: {int(value * 10)})")++ elif value == 0.0:+ result = guardrails.guard_input("test")+ print(f" β Guards executed: {len(result.guard_results)} (expected: 0)")++ elif value == 1 or value == 1.0:+ result = guardrails.guard_input("test")+ print(f" β Guards executed: {len(result.guard_results)} (expected: >0)")++ except Exception as e:+ print(f"β Sample rate {value} failed:")+ print(f" Error: {type(e).__name__}: {e}")+++if __name__ == "__main__":+ print("π§ͺ Testing Sample Rate Values")+ print("=" * 50)++ # Test all the requested values+ test_sample_rate(-0.2, "negative - should fail")+ test_sample_rate(0.2, "valid low - should work")+ test_sample_rate(0.0, "zero - should work")+ test_sample_rate(1.3, "too high - should fail")+ test_sample_rate(1, "one - should work")++ print(f"\nπ― Summary:")+ print("Expected results:")+ print("β 0.2, 0.0, 1 should work")+ print("β -0.2, 1.3 should fail with ValueError")\ No newline at end of file