Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

accumulate costs for attack enhancements and progressions

A-Vamshi100be0961ecd
diff --git a/deepteam/attacks/attack_engine/attack_engine.py b/deepteam/attacks/attack_engine/attack_engine.py
index bcd04c5a..a422ef1d 100644
--- a/deepteam/attacks/attack_engine/attack_engine.py
+++ b/deepteam/attacks/attack_engine/attack_engine.py
@@ -1,12 +1,16 @@
import asyncio
from enum import Enum
-from typing import List, Optional, Union
+from typing import List, Optional, Tuple, Union
from deepeval.models import DeepEvalBaseLLM
from deepeval.metrics.utils import initialize_model
from deepteam.test_case import RTTestCase
-from deepteam.attacks.attack_simulator.utils import generate, a_generate
+from deepteam.attacks.attack_simulator.utils import (
+ generate_with_cost,
+ a_generate_with_cost,
+ add_cost,
+)
from .template import AttackEngineTemplates
from .types import TransformedAttack, AttackVariations, ValidationResult
@@ -47,7 +51,7 @@ class AttackEngine:
vulnerability_type=vulnerability_type,
generation_guidelines=self.generation_guidelines,
)
- transformed: TransformedAttack = generate(
+ transformed, refine_cost = generate_with_cost(
transform_prompt, TransformedAttack, self.simulator_model
)
transformed_input = transformed.input.strip()
@@ -65,25 +69,37 @@ class AttackEngine:
generation_guidelines=self.generation_guidelines,
)
)
- variations: AttackVariations = generate(
+ variations, variation_cost = generate_with_cost(
variation_prompt, AttackVariations, self.simulator_model
)
+ refine_cost = add_cost(refine_cost, variation_cost)
candidates = [transformed_input] + [
item.strip() for item in variations.inputs
]
- valid_inputs = self._validate_candidates_with_llm(
+ valid_inputs, validation_cost = self._validate_candidates_with_llm(
candidates=candidates,
vulnerability=vulnerability,
vulnerability_type=vulnerability_type,
purpose=effective_purpose,
)
+ refine_cost = add_cost(refine_cost, validation_cost)
if not valid_inputs:
valid_inputs = [transformed_input]
+ # Carry the baseline simulation_cost forward and add this refine
+ # pass's cost, split evenly across the refined test cases produced.
+ combined_cost = add_cost(test_case.simulation_cost, refine_cost)
+ per_refined_cost = (
+ combined_cost / len(valid_inputs)
+ if combined_cost is not None and valid_inputs
+ else combined_cost
+ )
refined_test_cases.extend(
[
- self._clone_with_new_input(test_case, refined_input)
+ self._clone_with_new_input(
+ test_case, refined_input, per_refined_cost
+ )
for refined_input in valid_inputs
]
)
@@ -125,7 +141,7 @@ class AttackEngine:
vulnerability_type=vulnerability_type,
generation_guidelines=self.generation_guidelines,
)
- transformed: TransformedAttack = await a_generate(
+ transformed, refine_cost = await a_generate_with_cost(
transform_prompt, TransformedAttack, self.simulator_model
)
transformed_input = transformed.input.strip()
@@ -143,24 +159,38 @@ class AttackEngine:
generation_guidelines=self.generation_guidelines,
)
)
- variations: AttackVariations = await a_generate(
+ variations, variation_cost = await a_generate_with_cost(
variation_prompt, AttackVariations, self.simulator_model
)
+ refine_cost = add_cost(refine_cost, variation_cost)
candidates = [transformed_input] + [
item.strip() for item in variations.inputs
]
- valid_inputs = await self._a_validate_candidates_with_llm(
- candidates=candidates,
- vulnerability=vulnerability,
- vulnerability_type=vulnerability_type,
- purpose=effective_purpose,
+ valid_inputs, validation_cost = (
+ await self._a_validate_candidates_with_llm(
+ candidates=candidates,
+ vulnerability=vulnerability,
+ vulnerability_type=vulnerability_type,
+ purpose=effective_purpose,
+ )
)
+ refine_cost = add_cost(refine_cost, validation_cost)
if not valid_inputs:
valid_inputs = [transformed_input]
+ # Carry the baseline simulation_cost forward and add this refine pass's
+ # cost, split evenly across the refined test cases produced.
+ combined_cost = add_cost(test_case.simulation_cost, refine_cost)
+ per_refined_cost = (
+ combined_cost / len(valid_inputs)
+ if combined_cost is not None and valid_inputs
+ else combined_cost
+ )
return [
- self._clone_with_new_input(test_case, refined_input)
+ self._clone_with_new_input(
+ test_case, refined_input, per_refined_cost
+ )
for refined_input in valid_inputs
]
@@ -180,9 +210,10 @@ class AttackEngine:
vulnerability: str,
vulnerability_type: Optional[str],
purpose: Optional[str] = None,
- ) -> List[str]:
+ ) -> Tuple[List[str], Optional[float]]:
effective_purpose = purpose if purpose is not None else self.purpose
validated: List[str] = []
+ validation_cost = None
for candidate in candidates[: self.variations]:
if not candidate or not candidate.strip():
continue
@@ -194,15 +225,16 @@ class AttackEngine:
purpose=effective_purpose,
)
try:
- result: ValidationResult = generate(
+ result, cost = generate_with_cost(
prompt, ValidationResult, self.simulator_model
)
+ validation_cost = add_cost(validation_cost, cost)
if result.is_valid:
validated.append(candidate)
except Exception:
continue
- return validated
+ return validated, validation_cost
async def _a_validate_candidates_with_llm(
self,
@@ -210,13 +242,13 @@ class AttackEngine:
vulnerability: str,
vulnerability_type: Optional[str],
purpose: Optional[str] = None,
- ) -> List[str]:
+ ) -> Tuple[List[str], Optional[float]]:
effective_purpose = purpose if purpose is not None else self.purpose
trimmed = [c for c in candidates[: self.variations] if c and c.strip()]
if not trimmed:
return []
- async def validate_one(candidate: str) -> Optional[str]:
+ async def validate_one(candidate: str):
prompt = AttackEngineTemplates.validate_attack_template(
candidate_input=candidate,
vulnerability=vulnerability,
@@ -224,19 +256,25 @@ class AttackEngine:
purpose=effective_purpose,
)
try:
- result: ValidationResult = await a_generate(
+ result, cost = await a_generate_with_cost(
prompt, ValidationResult, self.simulator_model
)
- return candidate if result.is_valid else None
+ return (candidate if result.is_valid else None), cost
except Exception:
- return None
+ return None, None
results = await asyncio.gather(*[validate_one(c) for c in trimmed])
- return [c for c in results if c]
+ validated = [c for c, _ in results if c]
+ validation_cost = None
+ for _, cost in results:
+ validation_cost = add_cost(validation_cost, cost)
+ return validated, validation_cost
@staticmethod
def _clone_with_new_input(
- test_case: RTTestCase, refined_input: str
+ test_case: RTTestCase,
+ refined_input: str,
+ simulation_cost: Optional[float] = None,
) -> RTTestCase:
return RTTestCase(
vulnerability=test_case.vulnerability,
@@ -244,4 +282,5 @@ class AttackEngine:
input=refined_input,
attack_method=test_case.attack_method,
metadata=test_case.metadata,
+ simulation_cost=simulation_cost,
)
diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.py
index e6d2e73f..709d6707 100644
--- a/deepteam/attacks/attack_simulator/attack_simulator.py
+++ b/deepteam/attacks/attack_simulator/attack_simulator.py
@@ -15,6 +15,10 @@ from deepteam.attacks.multi_turn.types import CallbackType
from deepteam.errors import ModelRefusalError
from deepteam.test_case import RTTestCase
from deepteam.attacks.attack_engine import AttackEngine
+from deepteam.attacks.attack_simulator.utils import (
+ cost_accumulator,
+ add_cost,
+)
class BaselineAttack:
@@ -284,17 +288,74 @@ class AttackSimulator:
test_case.attack_method = attack.get_name()
turns = [RTTurn(role="user", content=attack_input)]
+ with cost_accumulator() as acc:
+ try:
+ res: List[RTTurn] = attack._get_turns(
+ model_callback=self.model_callback,
+ turns=turns,
+ vulnerability=test_case.vulnerability,
+ vulnerability_type=test_case.vulnerability_type.value,
+ simulator_model=self.simulator_model,
+ )
+
+ test_case.turns = res
+ test_case.actual_output = res[-1].content
+
+ except ModelRefusalError as e:
+ if ignore_errors:
+ test_case.error = e.message
+ return test_case
+ else:
+ raise
+ except Exception as e:
+ if ignore_errors:
+ test_case.error = (
+ f"Error enhancing multi-turn attack: {str(e)}"
+ )
+ return test_case
+ else:
+ raise
+ finally:
+ test_case.simulation_cost = add_cost(
+ test_case.simulation_cost, acc[0]
+ )
+
+ return test_case
+
+ # Regular attack handling (non-multi-turn)
+ attack_input = test_case.input
+ if attack_input is None:
+ return test_case
+
+ test_case.attack_method = attack.get_name()
+ sig = inspect.signature(attack.enhance)
+
+ with cost_accumulator() as acc:
try:
- res: List[RTTurn] = attack._get_turns(
- model_callback=self.model_callback,
- turns=turns,
- vulnerability=test_case.vulnerability,
- vulnerability_type=test_case.vulnerability_type.value,
- simulator_model=self.simulator_model,
- )
+ res = None
+ if (
+ "simulator_model" in sig.parameters
+ and "model_callback" in sig.parameters
+ ):
+ res = attack.enhance(
+ attack=attack_input,
+ simulator_model=self.simulator_model,
+ model_callback=self.model_callback,
+ )
+ elif "simulator_model" in sig.parameters:
+ res = attack.enhance(
+ attack=attack_input,
+ simulator_model=self.simulator_model,
+ )
+ elif "model_callback" in sig.parameters:
+ res = attack.enhance(
+ attack=attack_input,
+ model_callback=self.model_callback,
+ )
+ else:
+ res = attack.enhance(attack=attack_input)
- test_case.turns = res
- test_case.actual_output = res[-1].content
+ test_case.input = res
except ModelRefusalError as e:
if ignore_errors:
@@ -305,60 +366,15 @@ class AttackSimulator:
except Exception as e:
if ignore_errors:
test_case.error = (
- f"Error enhancing multi-turn attack: {str(e)}"
+ f"Error enhancing regular attack: {str(e)}"
)
return test_case
else:
raise
-
- return test_case
-
- # Regular attack handling (non-multi-turn)
- attack_input = test_case.input
- if attack_input is None:
- return test_case
-
- test_case.attack_method = attack.get_name()
- sig = inspect.signature(attack.enhance)
-
- try:
- res = None
- if (
- "simulator_model" in sig.parameters
- and "model_callback" in sig.parameters
- ):
- res = attack.enhance(
- attack=attack_input,
- simulator_model=self.simulator_model,
- model_callback=self.model_callback,
- )
- elif "simulator_model" in sig.parameters:
- res = attack.enhance(
- attack=attack_input,
- simulator_model=self.simulator_model,
+ finally:
+ test_case.simulation_cost = add_cost(
+ test_case.simulation_cost, acc[0]
)
- elif "model_callback" in sig.parameters:
- res = attack.enhance(
- attack=attack_input,
- model_callback=self.model_callback,
- )
- else:
- res = attack.enhance(attack=attack_input)
-
- test_case.input = res
-
- except ModelRefusalError as e:
- if ignore_errors:
- test_case.error = e.message
- return test_case
- else:
- raise
- except Exception as e:
- if ignore_errors:
- test_case.error = f"Error enhancing regular attack: {str(e)}"
- return test_case
- else:
- raise
return test_case
@@ -383,17 +399,71 @@ class AttackSimulator:
sig = inspect.signature(attack.a_enhance)
turns = [RTTurn(role="user", content=attack_input)]
+ with cost_accumulator() as acc:
+ try:
+ res: List[RTTurn] = await attack._a_get_turns(
+ model_callback=self.model_callback,
+ turns=turns,
+ vulnerability=test_case.vulnerability,
+ vulnerability_type=test_case.vulnerability_type.value,
+ simulator_model=self.simulator_model,
+ )
+
+ test_case.turns = res
+ test_case.actual_output = res[-1].content
+
+ except ModelRefusalError as e:
+ if ignore_errors:
+ test_case.error = e.message
+ return test_case
+ else:
+ raise
+ except:
+ if ignore_errors:
+ test_case.error = "Error enhancing attack"
+ return test_case
+ else:
+ raise
+ finally:
+ test_case.simulation_cost = add_cost(
+ test_case.simulation_cost, acc[0]
+ )
+
+ return test_case
+
+ attack_input = test_case.input
+ if attack_input is None:
+ return test_case
+
+ test_case.attack_method = attack.get_name()
+ sig = inspect.signature(attack.a_enhance)
+
+ with cost_accumulator() as acc:
try:
- res: List[RTTurn] = await attack._a_get_turns(
- model_callback=self.model_callback,
- turns=turns,
- vulnerability=test_case.vulnerability,
- vulnerability_type=test_case.vulnerability_type.value,
- simulator_model=self.simulator_model,
- )
+ res = None
+ if (
+ "simulator_model" in sig.parameters
+ and "model_callback" in sig.parameters
+ ):
+ res = await attack.a_enhance(
+ attack=attack_input,
+ simulator_model=self.simulator_model,
+ model_callback=self.model_callback,
+ )
+ elif "simulator_model" in sig.parameters:
+ res = await attack.a_enhance(
+ attack=attack_input,
+ simulator_model=self.simulator_model,
+ )
+ elif "model_callback" in sig.parameters:
+ res = await attack.a_enhance(
+ attack=attack_input,
+ model_callback=self.model_callback,
+ )
+ else:
+ res = await attack.a_enhance(attack=attack_input)
- test_case.turns = res
- test_case.actual_output = res[-1].content
+ test_case.input = res
except ModelRefusalError as e:
if ignore_errors:
@@ -407,53 +477,9 @@ class AttackSimulator:
return test_case
else:
raise
-
- return test_case
-
- attack_input = test_case.input
- if attack_input is None:
- return test_case
-
- test_case.attack_method = attack.get_name()
- sig = inspect.signature(attack.a_enhance)
-
- try:
- res = None
- if (
- "simulator_model" in sig.parameters
- and "model_callback" in sig.parameters
- ):
- res = await attack.a_enhance(
- attack=attack_input,
- simulator_model=self.simulator_model,
- model_callback=self.model_callback,
- )
- elif "simulator_model" in sig.parameters:
- res = await attack.a_enhance(
- attack=attack_input,
- simulator_model=self.simulator_model,
- )
- elif "model_callback" in sig.parameters:
- res = await attack.a_enhance(
- attack=attack_input,
- model_callback=self.model_callback,
+ finally:
+ test_case.simulation_cost = add_cost(
+ test_case.simulation_cost, acc[0]
)
- else:
- res = await attack.a_enhance(attack=attack_input)
-
- test_case.input = res
-
- except ModelRefusalError as e:
- if ignore_errors:
- test_case.error = e.message
- return test_case
- else:
- raise
- except:
- if ignore_errors:
- test_case.error = "Error enhancing attack"
- return test_case
- else:
- raise
return test_case
diff --git a/deepteam/attacks/attack_simulator/utils.py b/deepteam/attacks/attack_simulator/utils.py
index cd5d2d22..0e546a28 100644
--- a/deepteam/attacks/attack_simulator/utils.py
+++ b/deepteam/attacks/attack_simulator/utils.py
@@ -4,6 +4,8 @@ import os
import time
import asyncio
import logging
+from contextvars import ContextVar
+from contextlib import contextmanager
from deepeval.metrics.utils import trimAndLoadJson, initialize_model
from deepeval.models import DeepEvalBaseLLM
from deepteam.attacks.single_turn.escalation_constants import (
@@ -14,13 +16,55 @@ from deepteam.attacks.single_turn.escalation_constants import (
MAX_RETRIES = os.getenv("DEEPTEAM_MAX_RETRIES", 3)
-def generate(
+def add_cost(*costs):
+ """Combine simulation/evaluation costs where ``None`` means 'no cost
+ reported' (non-native models). Returns ``None`` only when every input is
+ ``None``; otherwise treats ``None`` as ``0``.
+ """
+ if all(c is None for c in costs):
+ return None
+ return sum(c or 0 for c in costs)
+
+
+# Context-local simulation-cost accumulator. When a ``cost_accumulator()``
+# scope is active, every ``generate``/``a_generate`` call records its cost into
+# it. This lets callers (e.g. attack enhancement) capture the cost of an entire
+# nested operation without threading cost through every method. ContextVars are
+# concurrency-safe: each asyncio task gets its own copy, and child tasks created
+# inside a scope share the same accumulator object.
+_cost_acc: ContextVar = ContextVar("deepteam_simulation_cost_acc", default=None)
+
+
+def _record_cost(cost):
+ """Add ``cost`` to the active accumulator, if any is in scope."""
+ acc = _cost_acc.get()
+ if acc is not None:
+ acc[0] = add_cost(acc[0], cost)
+
+
+@contextmanager
+def cost_accumulator():
+ """Open a context-local accumulator that captures the simulation cost of all
+ ``generate``/``a_generate`` calls made within the scope.
+
+ Yields a single-element list whose ``[0]`` holds the running total (``None``
+ until a native-model cost is recorded).
+ """
+ acc = [None]
+ token = _cost_acc.set(acc)
+ try:
+ yield acc
+ finally:
+ _cost_acc.reset(token)
+
+
+def generate_with_cost(
prompt: str,
schema: BaseModel,
model: DeepEvalBaseLLM = None,
-) -> BaseModel:
+):
"""
- Generate schema using the provided model with retry logic.
+ Generate schema using the provided model with retry logic, returning cost.
Args:
prompt: The prompt to send to the model
@@ -28,7 +72,9 @@ def generate(
model: The model to use
Returns:
- The validated schema object
+ A tuple ``(res, cost)`` where ``res`` is the validated schema object and
+ ``cost`` is the model-reported cost for native models, or ``None`` for
+ non-native models that do not report cost.
"""
_, using_native_model = initialize_model(model=model)
last_error = None
@@ -37,34 +83,40 @@ def generate(
for attempt in range(MAX_RETRIES):
try:
if using_native_model:
- res, _ = model.generate(prompt=retry_prompt, schema=schema)
+ res, cost = model.generate(prompt=retry_prompt, schema=schema)
if res is None:
- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")
- return res
+ raise ValueError(
+ "Model returned None. This could be because of your model's guardrails, please use a different model."
+ )
+ return res, cost
else:
try:
res = model.generate(prompt=retry_prompt, schema=schema)
if res is None:
- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")
+ raise ValueError(
+ "Model returned None. This could be because of your model's guardrails, please use a different model."
+ )
if isinstance(res, str):
data = trimAndLoadJson(res)
- return schema(**data)
+ return schema(**data), None
else:
- return res
+ return res, None
except TypeError:
res = model.generate(retry_prompt)
if res is None:
- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")
+ raise ValueError(
+ "Model returned None. This could be because of your model's guardrails, please use a different model."
+ )
data = trimAndLoadJson(res)
if schema == SyntheticDataList:
data_list = [
SyntheticData(**item) for item in data["data"]
]
- return SyntheticDataList(data=data_list)
+ return SyntheticDataList(data=data_list), None
else:
- return schema(**data)
+ return schema(**data), None
except Exception as e:
last_error = e
@@ -72,9 +124,7 @@ def generate(
escalated_prompt = (
f"{random_escalation_suffix(attempt)} \n\n {prompt}"
)
- retry_prompt = append_critic_feedback(
- escalated_prompt, str(e)
- )
+ retry_prompt = append_critic_feedback(escalated_prompt, str(e))
sleep_time = 2**attempt
logging.warning(
f"Generation failed on attempt {attempt + 1}. Retrying in {sleep_time}s... Error: {e}"
@@ -86,13 +136,27 @@ def generate(
)
-async def a_generate(
+def generate(
prompt: str,
schema: BaseModel,
- model: "DeepEvalBaseLLM" = None,
+ model: DeepEvalBaseLLM = None,
) -> BaseModel:
+ """Backward-compatible wrapper around :func:`generate_with_cost` that
+ returns only the validated schema object. Records cost into the active
+ :func:`cost_accumulator` scope, if any."""
+ res, cost = generate_with_cost(prompt, schema, model)
+ _record_cost(cost)
+ return res
+
+
+async def a_generate_with_cost(
+ prompt: str,
+ schema: BaseModel,
+ model: "DeepEvalBaseLLM" = None,
+):
"""
- Asynchronously generate schema using the provided model with retry logic.
+ Asynchronously generate schema using the provided model with retry logic,
+ returning cost.
Args:
prompt: The prompt to send to the model
@@ -100,7 +164,9 @@ async def a_generate(
model: The model to use
Returns:
- The validated schema object
+ A tuple ``(res, cost)`` where ``res`` is the validated schema object and
+ ``cost`` is the model-reported cost for native models, or ``None`` for
+ non-native models that do not report cost.
"""
_, using_native_model = initialize_model(model=model)
last_error = None
@@ -109,34 +175,44 @@ async def a_generate(
for attempt in range(MAX_RETRIES):
try:
if using_native_model:
- res, _ = await model.a_generate(prompt=retry_prompt, schema=schema)
+ res, cost = await model.a_generate(
+ prompt=retry_prompt, schema=schema
+ )
if res is None:
- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")
- return res
+ raise ValueError(
+ "Model returned None. This could be because of your model's guardrails, please use a different model."
+ )
+ return res, cost
else:
try:
- res = await model.a_generate(prompt=retry_prompt, schema=schema)
+ res = await model.a_generate(
+ prompt=retry_prompt, schema=schema
+ )
if res is None:
- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")
+ raise ValueError(
+ "Model returned None. This could be because of your model's guardrails, please use a different model."
+ )
if isinstance(res, str):
data = trimAndLoadJson(res)
- return schema(**data)
+ return schema(**data), None
else:
- return res
+ return res, None
except TypeError:
res = await model.a_generate(retry_prompt)
if res is None:
- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")
+ raise ValueError(
+ "Model returned None. This could be because of your model's guardrails, please use a different model."
+ )
data = trimAndLoadJson(res)
if schema == SyntheticDataList:
data_list = [
SyntheticData(**item) for item in data["data"]
]
- return SyntheticDataList(data=data_list)
+ return SyntheticDataList(data=data_list), None
else:
- return schema(**data)
+ return schema(**data), None
except Exception as e:
last_error = e
@@ -144,9 +220,7 @@ async def a_generate(
escalated_prompt = (
f"{random_escalation_suffix(attempt)} \n\n {prompt}"
)
- retry_prompt = append_critic_feedback(
- escalated_prompt, str(e)
- )
+ retry_prompt = append_critic_feedback(escalated_prompt, str(e))
sleep_time = 2**attempt
logging.warning(
f"Async generation failed on attempt {attempt + 1}. Retrying in {sleep_time}s... Error: {e}"
@@ -156,3 +230,16 @@ async def a_generate(
raise RuntimeError(
f"Failed to async generate after {MAX_RETRIES} attempts. Last error: {last_error}"
)
+
+
+async def a_generate(
+ prompt: str,
+ schema: BaseModel,
+ model: "DeepEvalBaseLLM" = None,
+) -> BaseModel:
+ """Backward-compatible wrapper around :func:`a_generate_with_cost` that
+ returns only the validated schema object. Records cost into the active
+ :func:`cost_accumulator` scope, if any."""
+ res, cost = await a_generate_with_cost(prompt, schema, model)
+ _record_cost(cost)
+ return res