DeepTeam is a framework to red team LLMs and AI agents.
Specify prompt kwarg to all model calls
Trevorc62fc3967ae0
diff --git a/deepteam/attacks/attack_simulator/utils.py b/deepteam/attacks/attack_simulator/utils.pyindex 0009c0fe..39768824 100644--- a/deepteam/attacks/attack_simulator/utils.py+++ b/deepteam/attacks/attack_simulator/utils.py@@ -23,11 +23,11 @@ def generate("""_, using_native_model = initialize_model(model=model)if using_native_model:- res, _ = model.generate(prompt, schema=schema)+ res, _ = model.generate(prompt=prompt, schema=schema)return reselse:try:- res = model.generate(prompt, schema=schema)+ res = model.generate(prompt=prompt, schema=schema)data = trimAndLoadJson(res)return schema(**data)except TypeError:@@ -59,11 +59,11 @@ async def a_generate(_, using_native_model = initialize_model(model=model)if using_native_model:- res, _ = await model.a_generate(prompt, schema=schema)+ res, _ = await model.a_generate(prompt=prompt, schema=schema)return reselse:try:- res = await model.a_generate(prompt, schema=schema)+ res = await model.a_generate(prompt=prompt, schema=schema)data = trimAndLoadJson(res)return schema(**data)except TypeError:diff --git a/deepteam/guardrails/guards/base_guard.py b/deepteam/guardrails/guards/base_guard.pyindex e100f907..957cd3f9 100644--- a/deepteam/guardrails/guards/base_guard.py+++ b/deepteam/guardrails/guards/base_guard.py@@ -40,7 +40,7 @@ class BaseGuard(ABC):self.evaluation_cost = 0 if self.using_native_model else Noneif self.using_native_model:res, cost = self.model.generate(- guard_prompt, schema=SafetyLevelSchema+ prompt=guard_prompt, schema=SafetyLevelSchema)self.evaluation_cost += costself.safety_level = res.safety_level@@ -71,7 +71,7 @@ class BaseGuard(ABC):self.evaluation_cost = 0 if self.using_native_model else Noneif self.using_native_model:res, cost = await self.model.a_generate(- guard_prompt, schema=SafetyLevelSchema+ prompt=guard_prompt, schema=SafetyLevelSchema)self.evaluation_cost += costself.safety_level = res.safety_leveldiff --git a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.pyindex c03f36a7..9de28b94 100644--- a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py+++ b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py@@ -93,13 +93,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):return self.purposeprompt = AmnesiaAssessmentTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):return self.purposeprompt = AmnesiaAssessmentTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py b/deepteam/metrics/agentic/boundary_violation/boundary_violation.pyindex 5b0d5c3b..3257934b 100644--- a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py+++ b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py@@ -93,13 +93,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):return self.purposeprompt = BoundaryViolationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):return self.purposeprompt = BoundaryViolationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/domain_validation/domain_validation.py b/deepteam/metrics/agentic/domain_validation/domain_validation.pyindex d28e9320..73d76372 100644--- a/deepteam/metrics/agentic/domain_validation/domain_validation.py+++ b/deepteam/metrics/agentic/domain_validation/domain_validation.py@@ -93,13 +93,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):return self.purposeprompt = DomainValidationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class DomainValidationMetric(BaseRedTeamingMetric):return self.purposeprompt = DomainValidationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class DomainValidationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/escalation_success/escalation_success.py b/deepteam/metrics/agentic/escalation_success/escalation_success.pyindex 01ecd25a..961ba9a8 100644--- a/deepteam/metrics/agentic/escalation_success/escalation_success.py+++ b/deepteam/metrics/agentic/escalation_success/escalation_success.py@@ -93,13 +93,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):return self.purposeprompt = EscalationSuccessTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):return self.purposeprompt = EscalationSuccessTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/extraction_success/extraction_success.py b/deepteam/metrics/agentic/extraction_success/extraction_success.pyindex d22801be..08ae8ecb 100644--- a/deepteam/metrics/agentic/extraction_success/extraction_success.py+++ b/deepteam/metrics/agentic/extraction_success/extraction_success.py@@ -93,13 +93,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):return self.purposeprompt = ExtractionSuccessTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):return self.purposeprompt = ExtractionSuccessTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/goal_drift/goal_drift.py b/deepteam/metrics/agentic/goal_drift/goal_drift.pyindex b634e59c..9cc5b320 100644--- a/deepteam/metrics/agentic/goal_drift/goal_drift.py+++ b/deepteam/metrics/agentic/goal_drift/goal_drift.py@@ -93,13 +93,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):return self.purposeprompt = GoalDriftTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class GoalDriftMetric(BaseRedTeamingMetric):return self.purposeprompt = GoalDriftTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class GoalDriftMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.pyindex ab0dc6eb..9d53e1af 100644--- a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py+++ b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py@@ -95,13 +95,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -116,12 +118,14 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -156,13 +162,13 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.pyindex 673241d2..8b9cd871 100644--- a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py+++ b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py@@ -95,13 +95,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -116,12 +118,14 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -156,13 +162,13 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py b/deepteam/metrics/agentic/leakage_rate/leakage_rate.pyindex 4765c808..efb59299 100644--- a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py+++ b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py@@ -93,13 +93,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):return self.purposeprompt = LeakageRateTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class LeakageRateMetric(BaseRedTeamingMetric):return self.purposeprompt = LeakageRateTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class LeakageRateMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.pyindex 91d294aa..01bef8d9 100644--- a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py+++ b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py@@ -95,13 +95,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -116,12 +118,14 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -156,13 +162,13 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py b/deepteam/metrics/agentic/misinterpretation/misinterpretation.pyindex 7f711abf..3a5368f0 100644--- a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py+++ b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py@@ -93,13 +93,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):return self.purposeprompt = MisinterpretationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class MisinterpretationMetric(BaseRedTeamingMetric):return self.purposeprompt = MisinterpretationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class MisinterpretationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.pyindex 8ffec3e7..f05605ab 100644--- a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py+++ b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py@@ -93,13 +93,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):return self.purposeprompt = PoisoningAssessmentTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):return self.purposeprompt = PoisoningAssessmentTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/subversion_success/subversion_success.py b/deepteam/metrics/agentic/subversion_success/subversion_success.pyindex e76ac4c6..a3c8010e 100644--- a/deepteam/metrics/agentic/subversion_success/subversion_success.py+++ b/deepteam/metrics/agentic/subversion_success/subversion_success.py@@ -93,13 +93,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):return self.purposeprompt = SubversionSuccessTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):return self.purposeprompt = SubversionSuccessTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.pyindex 1ac97566..054a3fc0 100644--- a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py+++ b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py@@ -93,13 +93,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):return self.purposeprompt = TemporalAssessmentTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):return self.purposeprompt = TemporalAssessmentTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.pyindex 35b0363b..054329ac 100644--- a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py+++ b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py@@ -95,13 +95,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -116,12 +118,14 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -156,13 +162,13 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py b/deepteam/metrics/agentic/verification_assessment/verification_assessment.pyindex 0216bc0c..0473b340 100644--- a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py+++ b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py@@ -95,13 +95,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -116,12 +118,14 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -156,13 +162,13 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/bfla/bfla.py b/deepteam/metrics/bfla/bfla.pyindex ff84238e..e2928426 100644--- a/deepteam/metrics/bfla/bfla.py+++ b/deepteam/metrics/bfla/bfla.py@@ -97,13 +97,15 @@ class BFLAMetric(BaseRedTeamingMetric):return self.purposeprompt = BFLATemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -116,12 +118,14 @@ class BFLAMetric(BaseRedTeamingMetric):return self.purposeprompt = BFLATemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -137,13 +141,15 @@ class BFLAMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -158,13 +164,13 @@ class BFLAMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/bias/bias.py b/deepteam/metrics/bias/bias.pyindex 83dd2e45..eaadbd21 100644--- a/deepteam/metrics/bias/bias.py+++ b/deepteam/metrics/bias/bias.py@@ -93,13 +93,15 @@ class BiasMetric(BaseRedTeamingMetric):return self.purposeprompt = BiasTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -112,12 +114,14 @@ class BiasMetric(BaseRedTeamingMetric):return self.purposeprompt = BiasTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -132,13 +136,15 @@ class BiasMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -152,13 +158,13 @@ class BiasMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/bola/bola.py b/deepteam/metrics/bola/bola.pyindex 9983a070..7522bd04 100644--- a/deepteam/metrics/bola/bola.py+++ b/deepteam/metrics/bola/bola.py@@ -98,13 +98,15 @@ class BOLAMetric(BaseRedTeamingMetric):async def a_generate_entities(self):prompt = BOLATemplate.extract_entities(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Entities)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Entities+ )self.evaluation_cost += costreturn res.entitieselse:try:res: Entities = await self.model.a_generate(- prompt, schema=Entities+ prompt=prompt, schema=Entities)return res.entitiesexcept TypeError:@@ -115,12 +117,14 @@ class BOLAMetric(BaseRedTeamingMetric):def generate_entities(self):prompt = BOLATemplate.extract_entities(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Entities)+ res, cost = self.model.generate(prompt=prompt, schema=Entities)self.evaluation_cost += costreturn res.entitieselse:try:- res: Entities = self.model.generate(prompt, schema=Entities)+ res: Entities = self.model.generate(+ prompt=prompt, schema=Entities+ )return res.entitiesexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class BOLAMetric(BaseRedTeamingMetric):entities=self.entities,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class BOLAMetric(BaseRedTeamingMetric):entities=self.entities,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/competitors/competitors.py b/deepteam/metrics/competitors/competitors.pyindex 8baace8d..785162fb 100644--- a/deepteam/metrics/competitors/competitors.py+++ b/deepteam/metrics/competitors/competitors.py@@ -96,13 +96,15 @@ class CompetitorsMetric(BaseRedTeamingMetric):return self.purposeprompt = CompetitorsTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -115,12 +117,14 @@ class CompetitorsMetric(BaseRedTeamingMetric):return self.purposeprompt = CompetitorsTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class CompetitorsMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class CompetitorsMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/contracts/contracts.py b/deepteam/metrics/contracts/contracts.pyindex 06b795a6..81015610 100644--- a/deepteam/metrics/contracts/contracts.py+++ b/deepteam/metrics/contracts/contracts.py@@ -91,13 +91,15 @@ class ContractsMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -110,13 +112,13 @@ class ContractsMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/debug_access/debug_access.py b/deepteam/metrics/debug_access/debug_access.pyindex a91cf93d..97665cc8 100644--- a/deepteam/metrics/debug_access/debug_access.py+++ b/deepteam/metrics/debug_access/debug_access.py@@ -96,13 +96,15 @@ class DebugAccessMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -115,13 +117,13 @@ class DebugAccessMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/excessive_agency/excessive_agency.py b/deepteam/metrics/excessive_agency/excessive_agency.pyindex 8295b935..16457817 100644--- a/deepteam/metrics/excessive_agency/excessive_agency.py+++ b/deepteam/metrics/excessive_agency/excessive_agency.py@@ -100,13 +100,15 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -121,13 +123,13 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/graphic_content/graphic_content.py b/deepteam/metrics/graphic_content/graphic_content.pyindex 563d9746..9093f880 100644--- a/deepteam/metrics/graphic_content/graphic_content.py+++ b/deepteam/metrics/graphic_content/graphic_content.py@@ -99,13 +99,15 @@ class GraphicMetric(BaseRedTeamingMetric):graphic_category=self.graphic_category,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -120,13 +122,13 @@ class GraphicMetric(BaseRedTeamingMetric):graphic_category=self.graphic_category,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/hallucination/hallucination.py b/deepteam/metrics/hallucination/hallucination.pyindex 31b3c441..33666e0f 100644--- a/deepteam/metrics/hallucination/hallucination.py+++ b/deepteam/metrics/hallucination/hallucination.py@@ -98,13 +98,15 @@ class HallucinationMetric(BaseRedTeamingMetric):return self.purposeprompt = HallucinationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -117,12 +119,14 @@ class HallucinationMetric(BaseRedTeamingMetric):return self.purposeprompt = HallucinationTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -137,13 +141,15 @@ class HallucinationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class HallucinationMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/harm/harm.py b/deepteam/metrics/harm/harm.pyindex 5eb80314..52965e68 100644--- a/deepteam/metrics/harm/harm.py+++ b/deepteam/metrics/harm/harm.py@@ -99,13 +99,15 @@ class HarmMetric(BaseRedTeamingMetric):harm_category=self.harm_category,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -120,13 +122,13 @@ class HarmMetric(BaseRedTeamingMetric):harm_category=self.harm_category,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/hijacking/hijacking.py b/deepteam/metrics/hijacking/hijacking.pyindex 97c19964..2d5ee24a 100644--- a/deepteam/metrics/hijacking/hijacking.py+++ b/deepteam/metrics/hijacking/hijacking.py@@ -96,13 +96,15 @@ class HijackingMetric(BaseRedTeamingMetric):return self.purposeprompt = HijackingTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -115,12 +117,14 @@ class HijackingMetric(BaseRedTeamingMetric):return self.purposeprompt = HijackingTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class HijackingMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class HijackingMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/illegal_activity/illegal_activity.py b/deepteam/metrics/illegal_activity/illegal_activity.pyindex 6e47ac7b..a8610305 100644--- a/deepteam/metrics/illegal_activity/illegal_activity.py+++ b/deepteam/metrics/illegal_activity/illegal_activity.py@@ -99,13 +99,15 @@ class IllegalMetric(BaseRedTeamingMetric):illegal_category=self.illegal_category,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -120,13 +122,13 @@ class IllegalMetric(BaseRedTeamingMetric):illegal_category=self.illegal_category,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/imitation/imitation.py b/deepteam/metrics/imitation/imitation.pyindex e23b45c7..213de63a 100644--- a/deepteam/metrics/imitation/imitation.py+++ b/deepteam/metrics/imitation/imitation.py@@ -98,13 +98,15 @@ class ImitationMetric(BaseRedTeamingMetric):async def a_generate_entities(self):prompt = ImitationTemplate.extract_entities(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Entities)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Entities+ )self.evaluation_cost += costreturn res.entitieselse:try:res: Entities = await self.model.a_generate(- prompt, schema=Entities+ prompt=prompt, schema=Entities)return res.entitiesexcept TypeError:@@ -115,12 +117,14 @@ class ImitationMetric(BaseRedTeamingMetric):def generate_entities(self):prompt = ImitationTemplate.extract_entities(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Entities)+ res, cost = self.model.generate(prompt=prompt, schema=Entities)self.evaluation_cost += costreturn res.entitieselse:try:- res: Entities = self.model.generate(prompt, schema=Entities)+ res: Entities = self.model.generate(+ prompt=prompt, schema=Entities+ )return res.entitiesexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class ImitationMetric(BaseRedTeamingMetric):entities=self.entities,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class ImitationMetric(BaseRedTeamingMetric):entities=self.entities,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/intellectual_property/intellectual_property.py b/deepteam/metrics/intellectual_property/intellectual_property.pyindex 54d2a476..946b9c4f 100644--- a/deepteam/metrics/intellectual_property/intellectual_property.py+++ b/deepteam/metrics/intellectual_property/intellectual_property.py@@ -98,13 +98,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -119,12 +121,14 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -139,13 +143,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -159,13 +165,13 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/misinformation/misinformation.py b/deepteam/metrics/misinformation/misinformation.pyindex b57fb2d7..ff98f83f 100644--- a/deepteam/metrics/misinformation/misinformation.py+++ b/deepteam/metrics/misinformation/misinformation.py@@ -99,13 +99,15 @@ class MisinformationMetric(BaseRedTeamingMetric):misinformation_category=self.misinformation_category,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -120,13 +122,13 @@ class MisinformationMetric(BaseRedTeamingMetric):misinformation_category=self.misinformation_category,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/overreliance/overreliance.py b/deepteam/metrics/overreliance/overreliance.pyindex ba56855c..f340f23b 100644--- a/deepteam/metrics/overreliance/overreliance.py+++ b/deepteam/metrics/overreliance/overreliance.py@@ -96,13 +96,15 @@ class OverrelianceMetric(BaseRedTeamingMetric):return self.purposeprompt = OverrelianceTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -115,12 +117,14 @@ class OverrelianceMetric(BaseRedTeamingMetric):return self.purposeprompt = OverrelianceTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class OverrelianceMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class OverrelianceMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/personal_safety/personal_safety.py b/deepteam/metrics/personal_safety/personal_safety.pyindex b2b1ff5f..d48a11ee 100644--- a/deepteam/metrics/personal_safety/personal_safety.py+++ b/deepteam/metrics/personal_safety/personal_safety.py@@ -99,13 +99,15 @@ class SafetyMetric(BaseRedTeamingMetric):safety_category=self.safety_category,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -120,13 +122,13 @@ class SafetyMetric(BaseRedTeamingMetric):safety_category=self.safety_category,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/pii/pii.py b/deepteam/metrics/pii/pii.pyindex 6993ccc4..5129b23a 100644--- a/deepteam/metrics/pii/pii.py+++ b/deepteam/metrics/pii/pii.py@@ -102,13 +102,15 @@ class PIIMetric(BaseRedTeamingMetric):return self.purposeprompt = PIITemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -121,12 +123,14 @@ class PIIMetric(BaseRedTeamingMetric):return self.purposeprompt = PIITemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class PIIMetric(BaseRedTeamingMetric):async def a_generate_entities(self) -> List[str]:prompt = PIITemplate.extract_entities(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Entities)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Entities+ )self.evaluation_cost += costreturn res.entitieselse:try:res: Entities = await self.model.a_generate(- prompt, schema=Entities+ prompt=prompt, schema=Entities)return res.entitiesexcept TypeError:@@ -153,12 +159,14 @@ class PIIMetric(BaseRedTeamingMetric):def generate_entities(self) -> List[str]:prompt = PIITemplate.extract_entities(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Entities)+ res, cost = self.model.generate(prompt=prompt, schema=Entities)self.evaluation_cost += costreturn res.entitieselse:try:- res: Entities = self.model.generate(prompt, schema=Entities)+ res: Entities = self.model.generate(+ prompt=prompt, schema=Entities+ )return res.entitiesexcept TypeError:res = self.model.generate(prompt)@@ -175,13 +183,15 @@ class PIIMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -197,13 +207,13 @@ class PIIMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/prompt_extraction/prompt_extraction.py b/deepteam/metrics/prompt_extraction/prompt_extraction.pyindex be8201b7..cfbb209c 100644--- a/deepteam/metrics/prompt_extraction/prompt_extraction.py+++ b/deepteam/metrics/prompt_extraction/prompt_extraction.py@@ -96,13 +96,15 @@ class PromptExtractionMetric(BaseRedTeamingMetric):return self.purposeprompt = PromptExtractionTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -115,12 +117,14 @@ class PromptExtractionMetric(BaseRedTeamingMetric):return self.purposeprompt = PromptExtractionTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class PromptExtractionMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class PromptExtractionMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/rbac/rbac.py b/deepteam/metrics/rbac/rbac.pyindex 292c97c3..0db4dd2f 100644--- a/deepteam/metrics/rbac/rbac.py+++ b/deepteam/metrics/rbac/rbac.py@@ -104,13 +104,15 @@ class RBACMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -125,13 +127,13 @@ class RBACMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -144,13 +146,15 @@ class RBACMetric(BaseRedTeamingMetric):return self.purposeprompt = RBACTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -163,12 +167,14 @@ class RBACMetric(BaseRedTeamingMetric):return self.purposeprompt = RBACTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)diff --git a/deepteam/metrics/shell_injection/shell_injection.py b/deepteam/metrics/shell_injection/shell_injection.pyindex e85ad69d..33d787ef 100644--- a/deepteam/metrics/shell_injection/shell_injection.py+++ b/deepteam/metrics/shell_injection/shell_injection.py@@ -96,13 +96,15 @@ class ShellInjectionMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -115,13 +117,13 @@ class ShellInjectionMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/sql_injection/sql_injection.py b/deepteam/metrics/sql_injection/sql_injection.pyindex 335a41ae..5dad9437 100644--- a/deepteam/metrics/sql_injection/sql_injection.py+++ b/deepteam/metrics/sql_injection/sql_injection.py@@ -94,13 +94,15 @@ class SQLInjectionMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -113,13 +115,13 @@ class SQLInjectionMetric(BaseRedTeamingMetric):input=test_case.input, actual_output=test_case.actual_output)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/ssrf/ssrf.py b/deepteam/metrics/ssrf/ssrf.pyindex d50258fd..290d9bbb 100644--- a/deepteam/metrics/ssrf/ssrf.py+++ b/deepteam/metrics/ssrf/ssrf.py@@ -96,13 +96,15 @@ class SSRFMetric(BaseRedTeamingMetric):return self.purposeprompt = SSRFTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=Purpose)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=Purpose+ )self.evaluation_cost += costreturn res.purposeelse:try:res: Purpose = await self.model.a_generate(- prompt, schema=Purpose+ prompt=prompt, schema=Purpose)return res.purposeexcept TypeError:@@ -115,12 +117,14 @@ class SSRFMetric(BaseRedTeamingMetric):return self.purposeprompt = SSRFTemplate.extract_purpose(self.system_prompt)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=Purpose)+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)self.evaluation_cost += costreturn res.purposeelse:try:- res: Purpose = self.model.generate(prompt, schema=Purpose)+ res: Purpose = self.model.generate(+ prompt=prompt, schema=Purpose+ )return res.purposeexcept TypeError:res = self.model.generate(prompt)@@ -136,13 +140,15 @@ class SSRFMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -157,13 +163,13 @@ class SSRFMetric(BaseRedTeamingMetric):purpose=self.purpose,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:diff --git a/deepteam/metrics/toxicity/toxicity.py b/deepteam/metrics/toxicity/toxicity.pyindex d5f98da5..f46a0b22 100644--- a/deepteam/metrics/toxicity/toxicity.py+++ b/deepteam/metrics/toxicity/toxicity.py@@ -99,13 +99,15 @@ class ToxicityMetric(BaseRedTeamingMetric):toxicity_category=self.toxicity_category,)if self.using_native_model:- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)+ res, cost = await self.model.a_generate(+ prompt=prompt, schema=ReasonScore+ )self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = await self.model.a_generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError:@@ -120,13 +122,13 @@ class ToxicityMetric(BaseRedTeamingMetric):toxicity_category=self.toxicity_category,)if self.using_native_model:- res, cost = self.model.generate(prompt, schema=ReasonScore)+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)self.evaluation_cost += costreturn res.score, res.reasonelse:try:res: ReasonScore = self.model.generate(- prompt, schema=ReasonScore+ prompt=prompt, schema=ReasonScore)return res.score, res.reasonexcept TypeError: