Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

Specify prompt kwarg to all model calls

Trevorc62fc3967ae0
diff --git a/deepteam/attacks/attack_simulator/utils.py b/deepteam/attacks/attack_simulator/utils.py
index 0009c0fe..39768824 100644
--- a/deepteam/attacks/attack_simulator/utils.py
+++ b/deepteam/attacks/attack_simulator/utils.py
@@ -23,11 +23,11 @@ def generate(
"""
_, using_native_model = initialize_model(model=model)
if using_native_model:
- res, _ = model.generate(prompt, schema=schema)
+ res, _ = model.generate(prompt=prompt, schema=schema)
return res
else:
try:
- res = model.generate(prompt, schema=schema)
+ res = model.generate(prompt=prompt, schema=schema)
data = trimAndLoadJson(res)
return schema(**data)
except TypeError:
@@ -59,11 +59,11 @@ async def a_generate(
_, using_native_model = initialize_model(model=model)
if using_native_model:
- res, _ = await model.a_generate(prompt, schema=schema)
+ res, _ = await model.a_generate(prompt=prompt, schema=schema)
return res
else:
try:
- res = await model.a_generate(prompt, schema=schema)
+ res = await model.a_generate(prompt=prompt, schema=schema)
data = trimAndLoadJson(res)
return schema(**data)
except TypeError:
diff --git a/deepteam/guardrails/guards/base_guard.py b/deepteam/guardrails/guards/base_guard.py
index e100f907..957cd3f9 100644
--- a/deepteam/guardrails/guards/base_guard.py
+++ b/deepteam/guardrails/guards/base_guard.py
@@ -40,7 +40,7 @@ class BaseGuard(ABC):
self.evaluation_cost = 0 if self.using_native_model else None
if self.using_native_model:
res, cost = self.model.generate(
- guard_prompt, schema=SafetyLevelSchema
+ prompt=guard_prompt, schema=SafetyLevelSchema
)
self.evaluation_cost += cost
self.safety_level = res.safety_level
@@ -71,7 +71,7 @@ class BaseGuard(ABC):
self.evaluation_cost = 0 if self.using_native_model else None
if self.using_native_model:
res, cost = await self.model.a_generate(
- guard_prompt, schema=SafetyLevelSchema
+ prompt=guard_prompt, schema=SafetyLevelSchema
)
self.evaluation_cost += cost
self.safety_level = res.safety_level
diff --git a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
index c03f36a7..9de28b94 100644
--- a/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
+++ b/deepteam/metrics/agentic/amnesia_assessment/amnesia_assessment.py
@@ -93,13 +93,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
return self.purpose
prompt = AmnesiaAssessmentTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
return self.purpose
prompt = AmnesiaAssessmentTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class AmnesiaAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
index 5b0d5c3b..3257934b 100644
--- a/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
+++ b/deepteam/metrics/agentic/boundary_violation/boundary_violation.py
@@ -93,13 +93,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = BoundaryViolationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = BoundaryViolationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class BoundaryViolationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/domain_validation/domain_validation.py b/deepteam/metrics/agentic/domain_validation/domain_validation.py
index d28e9320..73d76372 100644
--- a/deepteam/metrics/agentic/domain_validation/domain_validation.py
+++ b/deepteam/metrics/agentic/domain_validation/domain_validation.py
@@ -93,13 +93,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = DomainValidationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class DomainValidationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = DomainValidationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class DomainValidationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class DomainValidationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/escalation_success/escalation_success.py b/deepteam/metrics/agentic/escalation_success/escalation_success.py
index 01ecd25a..961ba9a8 100644
--- a/deepteam/metrics/agentic/escalation_success/escalation_success.py
+++ b/deepteam/metrics/agentic/escalation_success/escalation_success.py
@@ -93,13 +93,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
return self.purpose
prompt = EscalationSuccessTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
return self.purpose
prompt = EscalationSuccessTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class EscalationSuccessMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/extraction_success/extraction_success.py b/deepteam/metrics/agentic/extraction_success/extraction_success.py
index d22801be..08ae8ecb 100644
--- a/deepteam/metrics/agentic/extraction_success/extraction_success.py
+++ b/deepteam/metrics/agentic/extraction_success/extraction_success.py
@@ -93,13 +93,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
return self.purpose
prompt = ExtractionSuccessTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
return self.purpose
prompt = ExtractionSuccessTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class ExtractionSuccessMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/goal_drift/goal_drift.py b/deepteam/metrics/agentic/goal_drift/goal_drift.py
index b634e59c..9cc5b320 100644
--- a/deepteam/metrics/agentic/goal_drift/goal_drift.py
+++ b/deepteam/metrics/agentic/goal_drift/goal_drift.py
@@ -93,13 +93,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):
return self.purpose
prompt = GoalDriftTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class GoalDriftMetric(BaseRedTeamingMetric):
return self.purpose
prompt = GoalDriftTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class GoalDriftMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class GoalDriftMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
index ab0dc6eb..9d53e1af 100644
--- a/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
+++ b/deepteam/metrics/agentic/hallucination_detection/hallucination_detection.py
@@ -95,13 +95,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -116,12 +118,14 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -156,13 +162,13 @@ class HallucinationDetectionMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
index 673241d2..8b9cd871 100644
--- a/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
+++ b/deepteam/metrics/agentic/hierarchy_consistency/hierarchy_consistency.py
@@ -95,13 +95,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -116,12 +118,14 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -156,13 +162,13 @@ class HierarchyConsistencyMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
index 4765c808..efb59299 100644
--- a/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
+++ b/deepteam/metrics/agentic/leakage_rate/leakage_rate.py
@@ -93,13 +93,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):
return self.purpose
prompt = LeakageRateTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class LeakageRateMetric(BaseRedTeamingMetric):
return self.purpose
prompt = LeakageRateTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class LeakageRateMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class LeakageRateMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
index 91d294aa..01bef8d9 100644
--- a/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
+++ b/deepteam/metrics/agentic/manipulation_assessment/manipulation_assessment.py
@@ -95,13 +95,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -116,12 +118,14 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -156,13 +162,13 @@ class ManipulationAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
index 7f711abf..3a5368f0 100644
--- a/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
+++ b/deepteam/metrics/agentic/misinterpretation/misinterpretation.py
@@ -93,13 +93,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = MisinterpretationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = MisinterpretationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class MisinterpretationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
index 8ffec3e7..f05605ab 100644
--- a/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
+++ b/deepteam/metrics/agentic/poisoning_assessment/poisoning_assessment.py
@@ -93,13 +93,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
return self.purpose
prompt = PoisoningAssessmentTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
return self.purpose
prompt = PoisoningAssessmentTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class PoisoningAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/subversion_success/subversion_success.py b/deepteam/metrics/agentic/subversion_success/subversion_success.py
index e76ac4c6..a3c8010e 100644
--- a/deepteam/metrics/agentic/subversion_success/subversion_success.py
+++ b/deepteam/metrics/agentic/subversion_success/subversion_success.py
@@ -93,13 +93,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
return self.purpose
prompt = SubversionSuccessTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
return self.purpose
prompt = SubversionSuccessTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class SubversionSuccessMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
index 1ac97566..054a3fc0 100644
--- a/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
+++ b/deepteam/metrics/agentic/temporal_assessment/temporal_assessment.py
@@ -93,13 +93,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
return self.purpose
prompt = TemporalAssessmentTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
return self.purpose
prompt = TemporalAssessmentTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class TemporalAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
index 35b0363b..054329ac 100644
--- a/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
+++ b/deepteam/metrics/agentic/unauthorized_execution/unauthorized_execution.py
@@ -95,13 +95,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -116,12 +118,14 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -156,13 +162,13 @@ class UnauthorizedExecutionMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
index 0216bc0c..0473b340 100644
--- a/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
+++ b/deepteam/metrics/agentic/verification_assessment/verification_assessment.py
@@ -95,13 +95,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -116,12 +118,14 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -156,13 +162,13 @@ class VerificationAssessmentMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/bfla/bfla.py b/deepteam/metrics/bfla/bfla.py
index ff84238e..e2928426 100644
--- a/deepteam/metrics/bfla/bfla.py
+++ b/deepteam/metrics/bfla/bfla.py
@@ -97,13 +97,15 @@ class BFLAMetric(BaseRedTeamingMetric):
return self.purpose
prompt = BFLATemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -116,12 +118,14 @@ class BFLAMetric(BaseRedTeamingMetric):
return self.purpose
prompt = BFLATemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -137,13 +141,15 @@ class BFLAMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -158,13 +164,13 @@ class BFLAMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/bias/bias.py b/deepteam/metrics/bias/bias.py
index 83dd2e45..eaadbd21 100644
--- a/deepteam/metrics/bias/bias.py
+++ b/deepteam/metrics/bias/bias.py
@@ -93,13 +93,15 @@ class BiasMetric(BaseRedTeamingMetric):
return self.purpose
prompt = BiasTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -112,12 +114,14 @@ class BiasMetric(BaseRedTeamingMetric):
return self.purpose
prompt = BiasTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -132,13 +136,15 @@ class BiasMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -152,13 +158,13 @@ class BiasMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/bola/bola.py b/deepteam/metrics/bola/bola.py
index 9983a070..7522bd04 100644
--- a/deepteam/metrics/bola/bola.py
+++ b/deepteam/metrics/bola/bola.py
@@ -98,13 +98,15 @@ class BOLAMetric(BaseRedTeamingMetric):
async def a_generate_entities(self):
prompt = BOLATemplate.extract_entities(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Entities)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Entities
+ )
self.evaluation_cost += cost
return res.entities
else:
try:
res: Entities = await self.model.a_generate(
- prompt, schema=Entities
+ prompt=prompt, schema=Entities
)
return res.entities
except TypeError:
@@ -115,12 +117,14 @@ class BOLAMetric(BaseRedTeamingMetric):
def generate_entities(self):
prompt = BOLATemplate.extract_entities(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Entities)
+ res, cost = self.model.generate(prompt=prompt, schema=Entities)
self.evaluation_cost += cost
return res.entities
else:
try:
- res: Entities = self.model.generate(prompt, schema=Entities)
+ res: Entities = self.model.generate(
+ prompt=prompt, schema=Entities
+ )
return res.entities
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class BOLAMetric(BaseRedTeamingMetric):
entities=self.entities,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class BOLAMetric(BaseRedTeamingMetric):
entities=self.entities,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/competitors/competitors.py b/deepteam/metrics/competitors/competitors.py
index 8baace8d..785162fb 100644
--- a/deepteam/metrics/competitors/competitors.py
+++ b/deepteam/metrics/competitors/competitors.py
@@ -96,13 +96,15 @@ class CompetitorsMetric(BaseRedTeamingMetric):
return self.purpose
prompt = CompetitorsTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -115,12 +117,14 @@ class CompetitorsMetric(BaseRedTeamingMetric):
return self.purpose
prompt = CompetitorsTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class CompetitorsMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class CompetitorsMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/contracts/contracts.py b/deepteam/metrics/contracts/contracts.py
index 06b795a6..81015610 100644
--- a/deepteam/metrics/contracts/contracts.py
+++ b/deepteam/metrics/contracts/contracts.py
@@ -91,13 +91,15 @@ class ContractsMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -110,13 +112,13 @@ class ContractsMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/debug_access/debug_access.py b/deepteam/metrics/debug_access/debug_access.py
index a91cf93d..97665cc8 100644
--- a/deepteam/metrics/debug_access/debug_access.py
+++ b/deepteam/metrics/debug_access/debug_access.py
@@ -96,13 +96,15 @@ class DebugAccessMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -115,13 +117,13 @@ class DebugAccessMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/excessive_agency/excessive_agency.py b/deepteam/metrics/excessive_agency/excessive_agency.py
index 8295b935..16457817 100644
--- a/deepteam/metrics/excessive_agency/excessive_agency.py
+++ b/deepteam/metrics/excessive_agency/excessive_agency.py
@@ -100,13 +100,15 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -121,13 +123,13 @@ class ExcessiveAgencyMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/graphic_content/graphic_content.py b/deepteam/metrics/graphic_content/graphic_content.py
index 563d9746..9093f880 100644
--- a/deepteam/metrics/graphic_content/graphic_content.py
+++ b/deepteam/metrics/graphic_content/graphic_content.py
@@ -99,13 +99,15 @@ class GraphicMetric(BaseRedTeamingMetric):
graphic_category=self.graphic_category,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -120,13 +122,13 @@ class GraphicMetric(BaseRedTeamingMetric):
graphic_category=self.graphic_category,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/hallucination/hallucination.py b/deepteam/metrics/hallucination/hallucination.py
index 31b3c441..33666e0f 100644
--- a/deepteam/metrics/hallucination/hallucination.py
+++ b/deepteam/metrics/hallucination/hallucination.py
@@ -98,13 +98,15 @@ class HallucinationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = HallucinationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -117,12 +119,14 @@ class HallucinationMetric(BaseRedTeamingMetric):
return self.purpose
prompt = HallucinationTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -137,13 +141,15 @@ class HallucinationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class HallucinationMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/harm/harm.py b/deepteam/metrics/harm/harm.py
index 5eb80314..52965e68 100644
--- a/deepteam/metrics/harm/harm.py
+++ b/deepteam/metrics/harm/harm.py
@@ -99,13 +99,15 @@ class HarmMetric(BaseRedTeamingMetric):
harm_category=self.harm_category,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -120,13 +122,13 @@ class HarmMetric(BaseRedTeamingMetric):
harm_category=self.harm_category,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/hijacking/hijacking.py b/deepteam/metrics/hijacking/hijacking.py
index 97c19964..2d5ee24a 100644
--- a/deepteam/metrics/hijacking/hijacking.py
+++ b/deepteam/metrics/hijacking/hijacking.py
@@ -96,13 +96,15 @@ class HijackingMetric(BaseRedTeamingMetric):
return self.purpose
prompt = HijackingTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -115,12 +117,14 @@ class HijackingMetric(BaseRedTeamingMetric):
return self.purpose
prompt = HijackingTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class HijackingMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class HijackingMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/illegal_activity/illegal_activity.py b/deepteam/metrics/illegal_activity/illegal_activity.py
index 6e47ac7b..a8610305 100644
--- a/deepteam/metrics/illegal_activity/illegal_activity.py
+++ b/deepteam/metrics/illegal_activity/illegal_activity.py
@@ -99,13 +99,15 @@ class IllegalMetric(BaseRedTeamingMetric):
illegal_category=self.illegal_category,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -120,13 +122,13 @@ class IllegalMetric(BaseRedTeamingMetric):
illegal_category=self.illegal_category,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/imitation/imitation.py b/deepteam/metrics/imitation/imitation.py
index e23b45c7..213de63a 100644
--- a/deepteam/metrics/imitation/imitation.py
+++ b/deepteam/metrics/imitation/imitation.py
@@ -98,13 +98,15 @@ class ImitationMetric(BaseRedTeamingMetric):
async def a_generate_entities(self):
prompt = ImitationTemplate.extract_entities(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Entities)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Entities
+ )
self.evaluation_cost += cost
return res.entities
else:
try:
res: Entities = await self.model.a_generate(
- prompt, schema=Entities
+ prompt=prompt, schema=Entities
)
return res.entities
except TypeError:
@@ -115,12 +117,14 @@ class ImitationMetric(BaseRedTeamingMetric):
def generate_entities(self):
prompt = ImitationTemplate.extract_entities(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Entities)
+ res, cost = self.model.generate(prompt=prompt, schema=Entities)
self.evaluation_cost += cost
return res.entities
else:
try:
- res: Entities = self.model.generate(prompt, schema=Entities)
+ res: Entities = self.model.generate(
+ prompt=prompt, schema=Entities
+ )
return res.entities
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class ImitationMetric(BaseRedTeamingMetric):
entities=self.entities,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class ImitationMetric(BaseRedTeamingMetric):
entities=self.entities,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/intellectual_property/intellectual_property.py b/deepteam/metrics/intellectual_property/intellectual_property.py
index 54d2a476..946b9c4f 100644
--- a/deepteam/metrics/intellectual_property/intellectual_property.py
+++ b/deepteam/metrics/intellectual_property/intellectual_property.py
@@ -98,13 +98,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -119,12 +121,14 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
self.system_prompt
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -139,13 +143,15 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -159,13 +165,13 @@ class IntellectualPropertyMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/misinformation/misinformation.py b/deepteam/metrics/misinformation/misinformation.py
index b57fb2d7..ff98f83f 100644
--- a/deepteam/metrics/misinformation/misinformation.py
+++ b/deepteam/metrics/misinformation/misinformation.py
@@ -99,13 +99,15 @@ class MisinformationMetric(BaseRedTeamingMetric):
misinformation_category=self.misinformation_category,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -120,13 +122,13 @@ class MisinformationMetric(BaseRedTeamingMetric):
misinformation_category=self.misinformation_category,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/overreliance/overreliance.py b/deepteam/metrics/overreliance/overreliance.py
index ba56855c..f340f23b 100644
--- a/deepteam/metrics/overreliance/overreliance.py
+++ b/deepteam/metrics/overreliance/overreliance.py
@@ -96,13 +96,15 @@ class OverrelianceMetric(BaseRedTeamingMetric):
return self.purpose
prompt = OverrelianceTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -115,12 +117,14 @@ class OverrelianceMetric(BaseRedTeamingMetric):
return self.purpose
prompt = OverrelianceTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class OverrelianceMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class OverrelianceMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/personal_safety/personal_safety.py b/deepteam/metrics/personal_safety/personal_safety.py
index b2b1ff5f..d48a11ee 100644
--- a/deepteam/metrics/personal_safety/personal_safety.py
+++ b/deepteam/metrics/personal_safety/personal_safety.py
@@ -99,13 +99,15 @@ class SafetyMetric(BaseRedTeamingMetric):
safety_category=self.safety_category,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -120,13 +122,13 @@ class SafetyMetric(BaseRedTeamingMetric):
safety_category=self.safety_category,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/pii/pii.py b/deepteam/metrics/pii/pii.py
index 6993ccc4..5129b23a 100644
--- a/deepteam/metrics/pii/pii.py
+++ b/deepteam/metrics/pii/pii.py
@@ -102,13 +102,15 @@ class PIIMetric(BaseRedTeamingMetric):
return self.purpose
prompt = PIITemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -121,12 +123,14 @@ class PIIMetric(BaseRedTeamingMetric):
return self.purpose
prompt = PIITemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class PIIMetric(BaseRedTeamingMetric):
async def a_generate_entities(self) -> List[str]:
prompt = PIITemplate.extract_entities(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Entities)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Entities
+ )
self.evaluation_cost += cost
return res.entities
else:
try:
res: Entities = await self.model.a_generate(
- prompt, schema=Entities
+ prompt=prompt, schema=Entities
)
return res.entities
except TypeError:
@@ -153,12 +159,14 @@ class PIIMetric(BaseRedTeamingMetric):
def generate_entities(self) -> List[str]:
prompt = PIITemplate.extract_entities(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Entities)
+ res, cost = self.model.generate(prompt=prompt, schema=Entities)
self.evaluation_cost += cost
return res.entities
else:
try:
- res: Entities = self.model.generate(prompt, schema=Entities)
+ res: Entities = self.model.generate(
+ prompt=prompt, schema=Entities
+ )
return res.entities
except TypeError:
res = self.model.generate(prompt)
@@ -175,13 +183,15 @@ class PIIMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -197,13 +207,13 @@ class PIIMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/prompt_extraction/prompt_extraction.py b/deepteam/metrics/prompt_extraction/prompt_extraction.py
index be8201b7..cfbb209c 100644
--- a/deepteam/metrics/prompt_extraction/prompt_extraction.py
+++ b/deepteam/metrics/prompt_extraction/prompt_extraction.py
@@ -96,13 +96,15 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
return self.purpose
prompt = PromptExtractionTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -115,12 +117,14 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
return self.purpose
prompt = PromptExtractionTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class PromptExtractionMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/rbac/rbac.py b/deepteam/metrics/rbac/rbac.py
index 292c97c3..0db4dd2f 100644
--- a/deepteam/metrics/rbac/rbac.py
+++ b/deepteam/metrics/rbac/rbac.py
@@ -104,13 +104,15 @@ class RBACMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -125,13 +127,13 @@ class RBACMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -144,13 +146,15 @@ class RBACMetric(BaseRedTeamingMetric):
return self.purpose
prompt = RBACTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -163,12 +167,14 @@ class RBACMetric(BaseRedTeamingMetric):
return self.purpose
prompt = RBACTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
diff --git a/deepteam/metrics/shell_injection/shell_injection.py b/deepteam/metrics/shell_injection/shell_injection.py
index e85ad69d..33d787ef 100644
--- a/deepteam/metrics/shell_injection/shell_injection.py
+++ b/deepteam/metrics/shell_injection/shell_injection.py
@@ -96,13 +96,15 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -115,13 +117,13 @@ class ShellInjectionMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/sql_injection/sql_injection.py b/deepteam/metrics/sql_injection/sql_injection.py
index 335a41ae..5dad9437 100644
--- a/deepteam/metrics/sql_injection/sql_injection.py
+++ b/deepteam/metrics/sql_injection/sql_injection.py
@@ -94,13 +94,15 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -113,13 +115,13 @@ class SQLInjectionMetric(BaseRedTeamingMetric):
input=test_case.input, actual_output=test_case.actual_output
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/ssrf/ssrf.py b/deepteam/metrics/ssrf/ssrf.py
index d50258fd..290d9bbb 100644
--- a/deepteam/metrics/ssrf/ssrf.py
+++ b/deepteam/metrics/ssrf/ssrf.py
@@ -96,13 +96,15 @@ class SSRFMetric(BaseRedTeamingMetric):
return self.purpose
prompt = SSRFTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=Purpose)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=Purpose
+ )
self.evaluation_cost += cost
return res.purpose
else:
try:
res: Purpose = await self.model.a_generate(
- prompt, schema=Purpose
+ prompt=prompt, schema=Purpose
)
return res.purpose
except TypeError:
@@ -115,12 +117,14 @@ class SSRFMetric(BaseRedTeamingMetric):
return self.purpose
prompt = SSRFTemplate.extract_purpose(self.system_prompt)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=Purpose)
+ res, cost = self.model.generate(prompt=prompt, schema=Purpose)
self.evaluation_cost += cost
return res.purpose
else:
try:
- res: Purpose = self.model.generate(prompt, schema=Purpose)
+ res: Purpose = self.model.generate(
+ prompt=prompt, schema=Purpose
+ )
return res.purpose
except TypeError:
res = self.model.generate(prompt)
@@ -136,13 +140,15 @@ class SSRFMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -157,13 +163,13 @@ class SSRFMetric(BaseRedTeamingMetric):
purpose=self.purpose,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
diff --git a/deepteam/metrics/toxicity/toxicity.py b/deepteam/metrics/toxicity/toxicity.py
index d5f98da5..f46a0b22 100644
--- a/deepteam/metrics/toxicity/toxicity.py
+++ b/deepteam/metrics/toxicity/toxicity.py
@@ -99,13 +99,15 @@ class ToxicityMetric(BaseRedTeamingMetric):
toxicity_category=self.toxicity_category,
)
if self.using_native_model:
- res, cost = await self.model.a_generate(prompt, schema=ReasonScore)
+ res, cost = await self.model.a_generate(
+ prompt=prompt, schema=ReasonScore
+ )
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = await self.model.a_generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError:
@@ -120,13 +122,13 @@ class ToxicityMetric(BaseRedTeamingMetric):
toxicity_category=self.toxicity_category,
)
if self.using_native_model:
- res, cost = self.model.generate(prompt, schema=ReasonScore)
+ res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
self.evaluation_cost += cost
return res.score, res.reason
else:
try:
res: ReasonScore = self.model.generate(
- prompt, schema=ReasonScore
+ prompt=prompt, schema=ReasonScore
)
return res.score, res.reason
except TypeError: