Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

fixed cli issues

A-Vamshif61ab2a692e2
diff --git a/deepteam/cli/main.py b/deepteam/cli/main.py
index c24abcef..463c98aa 100644
--- a/deepteam/cli/main.py
+++ b/deepteam/cli/main.py
@@ -342,12 +342,20 @@ def run(
target_model_spec = target_cfg["model"]
target_model = load_model(target_model_spec)
- async def model_callback(input: str) -> str:
- response = await target_model.a_generate(input)
- # Ensure we return a string, handle different response types
- if isinstance(response, tuple):
- return str(response[0]) if response else "Empty response"
- return str(response)
+ if system_config.get("run_async", True):
+ async def model_callback(input: str, turns=None) -> str:
+ response = await target_model.a_generate(input)
+ # Ensure we return a string, handle different response types
+ if isinstance(response, tuple):
+ return str(response[0]) if response else "Empty response"
+ return str(response)
+ else:
+ def model_callback(input: str, turns=None) -> str:
+ response = target_model.generate(input)
+ # Ensure we return a string, handle different response types
+ if isinstance(response, tuple):
+ return str(response[0]) if response else "Empty response"
+ return str(response)
else:
raise ValueError(
@@ -362,8 +370,6 @@ def run(
ignore_errors=system_config.get("ignore_errors", False),
)
- red_teamer._print_risk_assessment()
-
# Save risk assessment if output folder is specified
file = None
if final_output_folder:
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index 7a9003d8..0b588be3 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -73,9 +73,9 @@ class RedTeamer:
vulnerabilities = framework.vulnerabilities
attacks = framework.attacks
else:
- if not vulnerabilities:
+ if not vulnerabilities or not attacks:
raise ValueError(
- "You must either provide a 'framework' or 'vulnerabilities'."
+ "You must either provide a 'framework' or 'vulnerabilities' and 'attacks."
)
if self.async_mode:
@@ -104,6 +104,8 @@ class RedTeamer:
if evaluation_model is not None:
self.evaluation_model = evaluation_model
+ if simulator_model is not None:
+ self.simulator_model = simulator_model
with capture_red_teamer_run(
vulnerabilities=[v.get_name() for v in vulnerabilities],
attacks=[a.get_name() for a in attacks],
@@ -123,7 +125,7 @@ class RedTeamer:
vulnerabilities=vulnerabilities,
attacks=attacks,
ignore_errors=ignore_errors,
- simulator_model=simulator_model,
+ simulator_model=self.simulator_model,
metadata=metadata,
)
)
@@ -211,13 +213,15 @@ class RedTeamer:
vulnerabilities = framework.vulnerabilities
attacks = framework.attacks
else:
- if not vulnerabilities:
+ if not vulnerabilities or not attacks:
raise ValueError(
- "You must either provide a 'framework' or 'vulnerabilities'."
+ "You must either provide a 'framework' or 'vulnerabilities' and 'attacks."
)
if evaluation_model is not None:
self.evaluation_model = evaluation_model
+ if simulator_model is not None:
+ self.simulator_model = simulator_model
with capture_red_teamer_run(
vulnerabilities=[v.get_name() for v in vulnerabilities],
@@ -237,7 +241,7 @@ class RedTeamer:
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
vulnerabilities=vulnerabilities,
attacks=attacks,
- simulator_model=simulator_model,
+ simulator_model=self.simulator_model,
ignore_errors=ignore_errors,
metadata=metadata,
)
@@ -332,10 +336,10 @@ class RedTeamer:
for _vulnerability in vulnerabilities:
if vulnerability_type in _vulnerability.types:
+ _vulnerability.evaluation_model = self.evaluation_model
metric: BaseRedTeamingMetric = _vulnerability._get_metric(
vulnerability_type
)
- metric.evaluation_model = self.evaluation_model
break
if multi_turn:
@@ -407,10 +411,10 @@ class RedTeamer:
for _vulnerability in vulnerabilities:
if vulnerability_type in _vulnerability.types:
+ _vulnerability.evaluation_model = self.evaluation_model
metric: BaseRedTeamingMetric = _vulnerability._get_metric(
vulnerability_type
)
- metric.evaluation_model = self.evaluation_model
break
if multi_turn: