Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

Update tests for sample rate validation and deterministic behavior verification

sid-murali1f2903f024da
diff --git a/deepteam/guardrails/guardrails.py b/deepteam/guardrails/guardrails.py
index 6951aebd..63fff38d 100644
--- a/deepteam/guardrails/guardrails.py
+++ b/deepteam/guardrails/guardrails.py
@@ -44,7 +44,7 @@ class Guardrails:
self.sample_rate = sample_rate
self.evaluation_model = evaluation_model
- self._request_count = 0 # Deterministic counter
+ self._request_count = 0
# Update all guards to use the specified evaluation model
self.input_guards = self._update_guards_model(input_guards, evaluation_model)
diff --git a/deepteam/guardrails/guards/cybersecurity_guard.py b/deepteam/guardrails/guards/cybersecurity_guard.py
index dccb692a..fc08027a 100644
--- a/deepteam/guardrails/guards/cybersecurity_guard.py
+++ b/deepteam/guardrails/guards/cybersecurity_guard.py
@@ -11,6 +11,7 @@ class CybersecurityGuard(BaseGuard):
purpose: Optional[str] = None,
categories: List[CyberattackCategory] = None,
guard_type: GuardType = GuardType.INPUT,
+ evaluation_model: Optional[str] = None,
):
super().__init__(evaluation_model=evaluation_model)
self.guard_type = guard_type
diff --git a/tests/test_input_output_guards.py b/tests/test_input_output_guards.py
index d8bcfa50..72957808 100644
--- a/tests/test_input_output_guards.py
+++ b/tests/test_input_output_guards.py
@@ -46,63 +46,93 @@ def test_evaluation_model():
print("βœ… Custom evaluation model works")
-def test_sample_rate():
- """Test sample rate functionality"""
+def test_sample_rate_negative():
+ """Test sample rate -0.2 (should fail)"""
from deepteam.guardrails import Guardrails
from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard
- # Test with 0% sample rate (should never run guards)
+ print("πŸ§ͺ Testing sample_rate = -0.2")
guardrails = Guardrails(
input_guards=[PromptInjectionGuard()],
output_guards=[ToxicityGuard()],
- sample_rate=0.0
+ sample_rate=-0.2
)
+ print("βœ… Sample rate -0.2 somehow worked")
+
+
+def test_sample_rate_valid_low():
+ """Test sample rate 0.2 (should work)"""
- result = guardrails.guard_input(input="Test input")
- assert len(result.guard_results) == 0 # Should be empty due to 0% sampling
- print("βœ… Sample rate 0.0 works (no guards executed)")
+ from deepteam.guardrails import Guardrails
+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard
- # Test with 100% sample rate (should always run guards)
+ print("πŸ§ͺ Testing sample_rate = 0.2")
guardrails = Guardrails(
input_guards=[PromptInjectionGuard()],
output_guards=[ToxicityGuard()],
- sample_rate=1.0
+ sample_rate=0.2
)
- result = guardrails.guard_input(input="Test input")
- assert len(result.guard_results) > 0 # Should have results due to 100% sampling
- print("βœ… Sample rate 1.0 works (guards executed)")
+ # Test deterministic behavior - should process every 5th request
+ processed = 0
+ for i in range(10):
+ result = guardrails.guard_input(f"test {i}")
+ if len(result.guard_results) > 0:
+ processed += 1
+
+ print(f"βœ… Sample rate 0.2 works - processed {processed}/10 requests")
-def test_sample_rate_validation():
- """Test sample rate validation"""
+def test_sample_rate_zero():
+ """Test sample rate 0.0 (should work)"""
from deepteam.guardrails import Guardrails
from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard
- # Test invalid sample rates
- try:
- Guardrails(
- input_guards=[PromptInjectionGuard()],
- output_guards=[ToxicityGuard()],
- sample_rate=1.5 # Invalid: > 1.0
- )
- assert False, "Should have raised ValueError"
- except ValueError:
- pass
+ print("πŸ§ͺ Testing sample_rate = 0.0")
+ guardrails = Guardrails(
+ input_guards=[PromptInjectionGuard()],
+ output_guards=[ToxicityGuard()],
+ sample_rate=0.0
+ )
- try:
- Guardrails(
- input_guards=[PromptInjectionGuard()],
- output_guards=[ToxicityGuard()],
- sample_rate=-0.1 # Invalid: < 0.0
- )
- assert False, "Should have raised ValueError"
- except ValueError:
- pass
+ result = guardrails.guard_input("test")
+ assert len(result.guard_results) == 0
+ print("βœ… Sample rate 0.0 works - no guards executed")
+
+
+def test_sample_rate_too_high():
+ """Test sample rate 1.3 (should fail)"""
- print("βœ… Sample rate validation works")
+ from deepteam.guardrails import Guardrails
+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard
+
+ print("πŸ§ͺ Testing sample_rate = 1.3")
+ guardrails = Guardrails(
+ input_guards=[PromptInjectionGuard()],
+ output_guards=[ToxicityGuard()],
+ sample_rate=1.3
+ )
+ print("βœ… Sample rate 1.3 somehow worked")
+
+
+def test_sample_rate_one():
+ """Test sample rate 1 (should work)"""
+
+ from deepteam.guardrails import Guardrails
+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard
+
+ print("πŸ§ͺ Testing sample_rate = 1")
+ guardrails = Guardrails(
+ input_guards=[PromptInjectionGuard()],
+ output_guards=[ToxicityGuard()],
+ sample_rate=1
+ )
+
+ result = guardrails.guard_input("test")
+ assert len(result.guard_results) > 0
+ print("βœ… Sample rate 1 works - guards executed")
def test_error_handling():
@@ -157,8 +187,9 @@ if __name__ == "__main__":
test_basic_functionality()
test_evaluation_model()
- test_sample_rate()
- test_sample_rate_validation()
+ test_sample_rate_valid_low() # Should work
+ test_sample_rate_zero() # Should work
+ test_sample_rate_one()
test_error_handling()
test_attack_detection()
diff --git a/tests/test_sample_rates.py b/tests/test_sample_rates.py
new file mode 100644
index 00000000..33380dd5
--- /dev/null
+++ b/tests/test_sample_rates.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Test different sample rate values to see error messages.
+"""
+
+def test_sample_rate(value, description):
+ """Test a specific sample rate value"""
+ print(f"\nπŸ§ͺ Testing sample_rate = {value} ({description})")
+
+ try:
+ from deepteam.guardrails import Guardrails
+ from deepteam.guardrails.guards import PromptInjectionGuard, ToxicityGuard
+
+ guardrails = Guardrails(
+ input_guards=[PromptInjectionGuard()],
+ output_guards=[ToxicityGuard()],
+ sample_rate=value
+ )
+
+ print(f"βœ… Sample rate {value} worked!")
+
+ # Quick test to see deterministic behavior
+ if value == 0.2:
+ processed = 0
+ for i in range(10):
+ result = guardrails.guard_input(f"test {i}")
+ if len(result.guard_results) > 0:
+ processed += 1
+ print(f" β†’ Processed {processed}/10 requests (expected: {int(value * 10)})")
+
+ elif value == 0.0:
+ result = guardrails.guard_input("test")
+ print(f" β†’ Guards executed: {len(result.guard_results)} (expected: 0)")
+
+ elif value == 1 or value == 1.0:
+ result = guardrails.guard_input("test")
+ print(f" β†’ Guards executed: {len(result.guard_results)} (expected: >0)")
+
+ except Exception as e:
+ print(f"❌ Sample rate {value} failed:")
+ print(f" Error: {type(e).__name__}: {e}")
+
+
+if __name__ == "__main__":
+ print("πŸ§ͺ Testing Sample Rate Values")
+ print("=" * 50)
+
+ # Test all the requested values
+ test_sample_rate(-0.2, "negative - should fail")
+ test_sample_rate(0.2, "valid low - should work")
+ test_sample_rate(0.0, "zero - should work")
+ test_sample_rate(1.3, "too high - should fail")
+ test_sample_rate(1, "one - should work")
+
+ print(f"\n🎯 Summary:")
+ print("Expected results:")
+ print("βœ… 0.2, 0.0, 1 should work")
+ print("❌ -0.2, 1.3 should fail with ValueError")
\ No newline at end of file