DeepTeam is a framework to red team LLMs and AI agents.
add evaluation examples and evaluation guidelines support
A-Vamshi70aed5bb9c4f
diff --git a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.pyindex 6a19a947..6b8492ba 100644--- a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py+++ b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.agent_identity_abuse import AgentIdentityAbuseTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import AgentIdentityAbuseMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -41,6 +42,8 @@ class AgentIdentityAbuse(BaseVulnerability):type.value for type in AgentIdentityAbuseType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=AgentIdentityAbuseType@@ -50,6 +53,8 @@ class AgentIdentityAbuse(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -289,6 +294,8 @@ class AgentIdentityAbuse(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.pyindex 9c9e73c3..6e8e1447 100644--- a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py+++ b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py@@ -13,6 +13,7 @@ from deepteam.vulnerabilities.autonomous_agent_drift import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import AutonomousAgentDriftMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -46,6 +47,8 @@ class AutonomousAgentDrift(BaseVulnerability):type.value for type in AutonomousAgentDriftType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=AutonomousAgentDriftType@@ -55,6 +58,8 @@ class AutonomousAgentDrift(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -294,6 +299,8 @@ class AutonomousAgentDrift(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex ff1793c2..8dd1a017 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.bfla import BFLATypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import BFLAMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -39,6 +40,8 @@ class BFLA(BaseVulnerability):evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",types: Optional[List[BFLALiteral]] = [type.value for type in BFLAType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=BFLAType@@ -48,6 +51,8 @@ class BFLA(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -281,6 +286,8 @@ class BFLA(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex 0ff4722c..478f74dd 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -11,6 +11,7 @@ from deepteam.attacks.base_attack import Exploitabilityfrom deepteam.vulnerabilities.bias import BiasTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import BiasMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -37,6 +38,8 @@ class Bias(BaseVulnerability):type.value for type in BiasType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=BiasType@@ -46,6 +49,8 @@ class Bias(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -284,6 +289,8 @@ class Bias(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex 156378a0..8f79493c 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.bola import BOLATypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import BOLAMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -39,6 +40,8 @@ class BOLA(BaseVulnerability):evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",types: Optional[List[BOLALiteral]] = [type.value for type in BOLAType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=BOLAType@@ -48,6 +51,8 @@ class BOLA(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -284,6 +289,8 @@ class BOLA(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.pyindex e671a476..64dae9ad 100644--- a/deepteam/vulnerabilities/child_protection/child_protection.py+++ b/deepteam/vulnerabilities/child_protection/child_protection.py@@ -10,6 +10,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.child_protection import ChildProtectionTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ChildProtectionMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -38,6 +39,8 @@ class ChildProtection(BaseVulnerability):type.value for type in ChildProtectionType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ChildProtectionType@@ -47,6 +50,8 @@ class ChildProtection(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class ChildProtection(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex e6c14e93..b14c60af 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -12,6 +12,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.competition import CompetitionTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import CompetitorsMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -43,6 +44,8 @@ class Competition(BaseVulnerability):type.value for type in CompetitionType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=CompetitionType@@ -52,6 +55,8 @@ class Competition(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -288,6 +293,8 @@ class Competition(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.pyindex 67d97867..3a308b43 100644--- a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py+++ b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py@@ -12,6 +12,7 @@ from deepteam.vulnerabilities.cross_context_retrieval import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import CrossContextRetrievalMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -38,6 +39,8 @@ class CrossContextRetrieval(BaseVulnerability):type.value for type in CrossContextRetrievalType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=CrossContextRetrievalType@@ -47,6 +50,8 @@ class CrossContextRetrieval(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -286,6 +291,8 @@ class CrossContextRetrieval(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex 2fdb4e82..678a8d11 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -10,6 +10,7 @@ from deepteam.utils import validate_model_callback_signaturefrom deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.metrics import BaseRedTeamingMetric, HarmMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.attack_simulator.schema import SyntheticDataListfrom deepteam.risks import getRiskCategory@@ -34,6 +35,8 @@ class CustomVulnerability(BaseVulnerability):Union[str, DeepEvalBaseLLM]] = "gpt-3.5-turbo-0125",evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):self.name = name@@ -53,6 +56,8 @@ class CustomVulnerability(BaseVulnerability):self.criteria = criteria.strip()self.simulator_model = simulator_modelself.evaluation_model = evaluation_model+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinesself.async_mode = async_modeself.verbose_mode = verbose_modeself.metric = None@@ -303,6 +308,8 @@ class CustomVulnerability(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)return self.metricdiff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex 19fbafdb..e1d6d797 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.debug_access import DebugAccessTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import DebugAccessMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -41,6 +42,8 @@ class DebugAccess(BaseVulnerability):type.value for type in DebugAccessType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=DebugAccessType@@ -50,6 +53,8 @@ class DebugAccess(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -283,6 +288,8 @@ class DebugAccess(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.pyindex 67a31f8b..9d21df94 100644--- a/deepteam/vulnerabilities/ethics/ethics.py+++ b/deepteam/vulnerabilities/ethics/ethics.py@@ -10,6 +10,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.ethics import EthicsTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import EthicsMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -38,6 +39,8 @@ class Ethics(BaseVulnerability):type.value for type in EthicsType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=EthicsType@@ -47,6 +50,8 @@ class Ethics(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class Ethics(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.pyindex 9c9aec33..d707413a 100644--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py@@ -14,6 +14,7 @@ from deepteam.vulnerabilities.excessive_agency import (from deepteam.test_case import RTTestCasefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ExcessiveAgencyMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.attack_simulator.schema import SyntheticDataListfrom .template import ExcessiveAgencyTemplate@@ -38,6 +39,8 @@ class ExcessiveAgency(BaseVulnerability):type.value for type in ExcessiveAgencyType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ExcessiveAgencyType@@ -47,6 +50,8 @@ class ExcessiveAgency(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -282,6 +287,8 @@ class ExcessiveAgency(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.pyindex 5b2ca0bc..a9c029c4 100644--- a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py+++ b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py@@ -10,6 +10,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.exploit_tool_agent import ExploitToolAgentTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ExploitToolAgentMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -38,6 +39,8 @@ class ExploitToolAgent(BaseVulnerability):type.value for type in ExploitToolAgentType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ExploitToolAgentType@@ -47,6 +50,8 @@ class ExploitToolAgent(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -286,6 +291,8 @@ class ExploitToolAgent(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.pyindex f3f6c170..94550e97 100644--- a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py+++ b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py@@ -12,6 +12,7 @@ from deepteam.vulnerabilities.external_system_abuse import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ExternalSystemAbuseMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -40,6 +41,8 @@ class ExternalSystemAbuse(BaseVulnerability):type.value for type in ExternalSystemAbuseType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ExternalSystemAbuseType@@ -49,6 +52,8 @@ class ExternalSystemAbuse(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -287,6 +292,8 @@ class ExternalSystemAbuse(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.pyindex 87b36dcd..5d48725f 100644--- a/deepteam/vulnerabilities/fairness/fairness.py+++ b/deepteam/vulnerabilities/fairness/fairness.py@@ -10,6 +10,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.fairness import FairnessTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import FairnessMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -38,6 +39,8 @@ class Fairness(BaseVulnerability):type.value for type in FairnessType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=FairnessType@@ -47,6 +50,8 @@ class Fairness(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class Fairness(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.pyindex afe41109..17c3ed98 100644--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py@@ -1,3 +1,4 @@+from deepteam.metrics.types import EvaluationExamplefrom typing import List, Literal, Optional, Union, Dictimport asyncio@@ -41,6 +42,8 @@ class GoalTheft(BaseVulnerability):type.value for type in GoalTheftType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=GoalTheftType@@ -50,6 +53,8 @@ class GoalTheft(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class GoalTheft(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 761c38f8..68f68e01 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.graphic_content import GraphicContentTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import GraphicMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -39,6 +40,8 @@ class GraphicContent(BaseVulnerability):type.value for type in GraphicContentType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=GraphicContentType@@ -48,6 +51,8 @@ class GraphicContent(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -286,6 +291,8 @@ class GraphicContent(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/hallucination/hallucination.py b/deepteam/vulnerabilities/hallucination/hallucination.pyindex 638344ea..ed9eae6a 100644--- a/deepteam/vulnerabilities/hallucination/hallucination.py+++ b/deepteam/vulnerabilities/hallucination/hallucination.py@@ -11,6 +11,7 @@ from deepteam.attacks.base_attack import Exploitabilityfrom deepteam.vulnerabilities.hallucination.types import HallucinationTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import HallucinationMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -42,6 +43,8 @@ class Hallucination(BaseVulnerability):type.value for type in HallucinationType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=HallucinationType@@ -51,6 +54,8 @@ class Hallucination(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -280,6 +285,8 @@ class Hallucination(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex d1e2c583..0c1bd244 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.illegal_activity import IllegalActivityTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import IllegalMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -45,6 +46,8 @@ class IllegalActivity(BaseVulnerability):type.value for type in IllegalActivityType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=IllegalActivityType@@ -54,6 +57,8 @@ class IllegalActivity(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -293,6 +298,8 @@ class IllegalActivity(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.pyindex 46523fa1..f81f2c26 100644--- a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py+++ b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py@@ -13,6 +13,7 @@ from deepteam.vulnerabilities.indirect_instruction import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import IndirectInstructionMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -44,6 +45,8 @@ class IndirectInstruction(BaseVulnerability):type.value for type in IndirectInstructionType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=IndirectInstructionType@@ -53,6 +56,8 @@ class IndirectInstruction(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -292,6 +297,8 @@ class IndirectInstruction(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.pyindex f1b3b2bb..54933a7f 100644--- a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py+++ b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py@@ -16,6 +16,7 @@ from deepteam.metrics import (InsecureInterAgentCommunicationMetric,BaseRedTeamingMetric,)+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -44,6 +45,8 @@ class InsecureInterAgentCommunication(BaseVulnerability):type.value for type in InsecureInterAgentCommunicationType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(),@@ -55,6 +58,8 @@ class InsecureInterAgentCommunication(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -304,6 +309,8 @@ class InsecureInterAgentCommunication(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex 151c7cf1..83e50b40 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -13,6 +13,7 @@ from deepteam.vulnerabilities.intellectual_property import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import IntellectualPropertyMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -44,6 +45,8 @@ class IntellectualProperty(BaseVulnerability):type.value for type in IntellectualPropertyType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=IntellectualPropertyType@@ -53,6 +56,8 @@ class IntellectualProperty(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -290,6 +295,8 @@ class IntellectualProperty(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex 453f4f44..2c96873a 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.misinformation import MisinformationTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import MisinformationMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -39,6 +40,8 @@ class Misinformation(BaseVulnerability):type.value for type in MisinformationType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=MisinformationType@@ -48,6 +51,8 @@ class Misinformation(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class Misinformation(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex 59b00b1b..1dd25a4b 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.personal_safety import PersonalSafetyTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import SafetyMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -43,6 +44,8 @@ class PersonalSafety(BaseVulnerability):type.value for type in PersonalSafetyType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=PersonalSafetyType@@ -52,6 +55,8 @@ class PersonalSafety(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class PersonalSafety(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex 01667983..1413b4ec 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.pii_leakage import PIILeakageTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import PIIMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -42,6 +43,8 @@ class PIILeakage(BaseVulnerability):type.value for type in PIILeakageType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=PIILeakageType@@ -51,6 +54,8 @@ class PIILeakage(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -291,6 +296,8 @@ class PIILeakage(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex c0848e6b..6bf4482d 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.prompt_leakage import PromptLeakageTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import PromptExtractionMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -42,6 +43,8 @@ class PromptLeakage(BaseVulnerability):type.value for type in PromptLeakageType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=PromptLeakageType@@ -51,6 +54,8 @@ class PromptLeakage(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -288,6 +293,8 @@ class PromptLeakage(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex ca2ea6ac..adc4b669 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.rbac import RBACTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import RBACMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -39,6 +40,8 @@ class RBAC(BaseVulnerability):evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",types: Optional[List[RBACLiteral]] = [type.value for type in RBACType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=RBACType@@ -48,6 +51,8 @@ class RBAC(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -285,6 +290,8 @@ class RBAC(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.pyindex 8fa45042..bbb7f0d7 100644--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py@@ -1,3 +1,4 @@+from deepteam.metrics.types import EvaluationExamplefrom typing import List, Literal, Optional, Union, Dictimport asyncio@@ -45,6 +46,8 @@ class RecursiveHijacking(BaseVulnerability):type.value for type in RecursiveHijackingType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=RecursiveHijackingType@@ -54,6 +57,8 @@ class RecursiveHijacking(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -289,6 +294,8 @@ class RecursiveHijacking(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.pyindex bfe27891..7320265d 100644--- a/deepteam/vulnerabilities/robustness/robustness.py+++ b/deepteam/vulnerabilities/robustness/robustness.py@@ -15,6 +15,7 @@ from deepteam.metrics import (OverrelianceMetric,BaseRedTeamingMetric,)+from deepteam.metrics.types import EvaluationExamplefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -40,6 +41,8 @@ class Robustness(BaseVulnerability):type.value for type in RobustnessType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=RobustnessType@@ -49,6 +52,8 @@ class Robustness(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -286,6 +291,8 @@ class Robustness(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)if type == RobustnessType.INPUT_OVERRELIANCE:return OverrelianceMetric(@@ -293,6 +300,8 @@ class Robustness(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)raise ValueError("Invalid type passed in the 'get_metric' function. Please pass an enum from 'RobustnessType'"diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex 5680de76..b03c6618 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.shell_injection import ShellInjectionTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ShellInjectionMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -41,6 +42,8 @@ class ShellInjection(BaseVulnerability):type.value for type in ShellInjectionType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ShellInjectionType@@ -50,6 +53,8 @@ class ShellInjection(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -280,6 +285,8 @@ class ShellInjection(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex 55d8a7cc..41191b99 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.sql_injection import SQLInjectionTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import SQLInjectionMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -41,6 +42,8 @@ class SQLInjection(BaseVulnerability):type.value for type in SQLInjectionType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=SQLInjectionType@@ -50,6 +53,8 @@ class SQLInjection(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -280,6 +285,8 @@ class SQLInjection(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex 3077e38d..139f275a 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.ssrf import SSRFTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import SSRFMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -39,6 +40,8 @@ class SSRF(BaseVulnerability):evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",types: Optional[List[SSRFLiteral]] = [type.value for type in SSRFType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=SSRFType@@ -48,6 +51,8 @@ class SSRF(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -279,6 +284,8 @@ class SSRF(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.pyindex 011e873e..3b382708 100644--- a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py+++ b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py@@ -12,6 +12,7 @@ from deepteam.vulnerabilities.system_reconnaissance import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import SystemReconnaissanceMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -40,6 +41,8 @@ class SystemReconnaissance(BaseVulnerability):type.value for type in SystemReconnaissanceType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=SystemReconnaissanceType@@ -49,6 +52,8 @@ class SystemReconnaissance(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -288,6 +293,8 @@ class SystemReconnaissance(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.pyindex b7c0cb3e..4cd17f9f 100644--- a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py+++ b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py@@ -13,6 +13,7 @@ from deepteam.vulnerabilities.tool_metadata_poisoning import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ToolMetadataPoisoningMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -44,6 +45,8 @@ class ToolMetadataPoisoning(BaseVulnerability):type.value for type in ToolMetadataPoisoningType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ToolMetadataPoisoningType@@ -53,6 +56,8 @@ class ToolMetadataPoisoning(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -292,6 +297,8 @@ class ToolMetadataPoisoning(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.pyindex ddf238d3..919bdb11 100644--- a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py+++ b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py@@ -13,6 +13,7 @@ from deepteam.vulnerabilities.tool_orchestration_abuse import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ToolOrchestrationMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -44,6 +45,8 @@ class ToolOrchestrationAbuse(BaseVulnerability):type.value for type in ToolOrchestrationAbuseType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(),@@ -55,6 +58,8 @@ class ToolOrchestrationAbuse(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -294,6 +299,8 @@ class ToolOrchestrationAbuse(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex 8b05bd52..84eb796b 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -11,6 +11,7 @@ from deepteam.vulnerabilities import BaseVulnerabilityfrom deepteam.vulnerabilities.toxicity import ToxicityTypefrom deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import ToxicityMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -37,6 +38,8 @@ class Toxicity(BaseVulnerability):type.value for type in ToxicityType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(), types=types, allowed_type=ToxicityType@@ -46,6 +49,8 @@ class Toxicity(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -281,6 +286,8 @@ class Toxicity(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool:diff --git a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.pyindex aa4e29d7..ad7ca2f7 100644--- a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py+++ b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py@@ -13,6 +13,7 @@ from deepteam.vulnerabilities.unexpected_code_execution import ()from deepteam.vulnerabilities.utils import validate_vulnerability_typesfrom deepteam.metrics import UnexpectedCodeExecutionMetric, BaseRedTeamingMetric+from deepteam.metrics.types import EvaluationExamplefrom deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_simulator.schema import SyntheticDataList@@ -41,6 +42,8 @@ class UnexpectedCodeExecution(BaseVulnerability):type.value for type in UnexpectedCodeExecutionType],purpose: Optional[str] = None,+ evaluation_examples: Optional[List[EvaluationExample]] = None,+ evaluation_guidelines: Optional[List[str]] = None,):enum_types = validate_vulnerability_types(self.get_name(),@@ -52,6 +55,8 @@ class UnexpectedCodeExecution(BaseVulnerability):self.simulator_model = simulator_modelself.evaluation_model = evaluation_modelself.purpose = purpose+ self.evaluation_examples = evaluation_examples+ self.evaluation_guidelines = evaluation_guidelinessuper().__init__(types=enum_types)def assess(@@ -295,6 +300,8 @@ class UnexpectedCodeExecution(BaseVulnerability):model=self.evaluation_model,async_mode=self.async_mode,verbose_mode=self.verbose_mode,+ evaluation_examples=self.evaluation_examples,+ evaluation_guidelines=self.evaluation_guidelines,)def is_vulnerable(self) -> bool: