DeepTeam is a framework to red team LLMs and AI agents.
add context-aware retry logic to attack generations
A-Vamshif2fd59f1c0f3
diff --git a/deepteam/attacks/attack_simulator/utils.py b/deepteam/attacks/attack_simulator/utils.pyindex bdb0aa30..cd5d2d22 100644--- a/deepteam/attacks/attack_simulator/utils.py+++ b/deepteam/attacks/attack_simulator/utils.py@@ -6,6 +6,10 @@ import asyncioimport loggingfrom deepeval.metrics.utils import trimAndLoadJson, initialize_modelfrom deepeval.models import DeepEvalBaseLLM+from deepteam.attacks.single_turn.escalation_constants import (+ append_critic_feedback,+ random_escalation_suffix,+)MAX_RETRIES = os.getenv("DEEPTEAM_MAX_RETRIES", 3)@@ -28,19 +32,20 @@ def generate("""_, using_native_model = initialize_model(model=model)last_error = None+ retry_prompt = promptfor attempt in range(MAX_RETRIES):try:if using_native_model:- res, _ = model.generate(prompt=prompt, schema=schema)+ res, _ = model.generate(prompt=retry_prompt, schema=schema)if res is None:- raise ValueError("Model returned None.")+ raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")return reselse:try:- res = model.generate(prompt=prompt, schema=schema)+ res = model.generate(prompt=retry_prompt, schema=schema)if res is None:- raise ValueError("Model returned None.")+ raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")if isinstance(res, str):data = trimAndLoadJson(res)@@ -48,9 +53,9 @@ def generate(else:return resexcept TypeError:- res = model.generate(prompt)+ res = model.generate(retry_prompt)if res is None:- raise ValueError("Model returned None.")+ raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")data = trimAndLoadJson(res)if schema == SyntheticDataList:@@ -64,6 +69,12 @@ def generate(except Exception as e:last_error = eif attempt < MAX_RETRIES - 1:+ escalated_prompt = (+ f"{random_escalation_suffix(attempt)} \n\n {prompt}"+ )+ retry_prompt = append_critic_feedback(+ escalated_prompt, str(e)+ )sleep_time = 2**attemptlogging.warning(f"Generation failed on attempt {attempt + 1}. Retrying in {sleep_time}s... Error: {e}"@@ -93,19 +104,20 @@ async def a_generate("""_, using_native_model = initialize_model(model=model)last_error = None+ retry_prompt = promptfor attempt in range(MAX_RETRIES):try:if using_native_model:- res, _ = await model.a_generate(prompt=prompt, schema=schema)+ res, _ = await model.a_generate(prompt=retry_prompt, schema=schema)if res is None:- raise ValueError("Model returned None.")+ raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")return reselse:try:- res = await model.a_generate(prompt=prompt, schema=schema)+ res = await model.a_generate(prompt=retry_prompt, schema=schema)if res is None:- raise ValueError("Model returned None.")+ raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")if isinstance(res, str):data = trimAndLoadJson(res)@@ -113,9 +125,9 @@ async def a_generate(else:return resexcept TypeError:- res = await model.a_generate(prompt)+ res = await model.a_generate(retry_prompt)if res is None:- raise ValueError("Model returned None.")+ raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")data = trimAndLoadJson(res)if schema == SyntheticDataList:@@ -129,6 +141,12 @@ async def a_generate(except Exception as e:last_error = eif attempt < MAX_RETRIES - 1:+ escalated_prompt = (+ f"{random_escalation_suffix(attempt)} \n\n {prompt}"+ )+ retry_prompt = append_critic_feedback(+ escalated_prompt, str(e)+ )sleep_time = 2**attemptlogging.warning(f"Async generation failed on attempt {attempt + 1}. Retrying in {sleep_time}s... Error: {e}"diff --git a/deepteam/errors.py b/deepteam/errors.pyindex 8f632ad0..29bce0a6 100644--- a/deepteam/errors.py+++ b/deepteam/errors.py@@ -4,7 +4,7 @@ from typing import Optionalclass ModelRefusalError(Exception):def __init__(self, entity: Optional[str] = None):- self.message = "Simulator model refused to simulate harmful attacks"+ self.message = "Simulator model refused to simulate harmful attacks, this could be because of your model's guardrails, please use a different model."if entity:self.message += f" for '{entity}'"