DeepTeam is a framework to red team LLMs and AI agents.
accumulate costs for attack enhancements and progressions
A-Vamshi100be0961ecd
diff --git a/deepteam/attacks/attack_engine/attack_engine.py b/deepteam/attacks/attack_engine/attack_engine.pyindex bcd04c5a..a422ef1d 100644--- a/deepteam/attacks/attack_engine/attack_engine.py+++ b/deepteam/attacks/attack_engine/attack_engine.py@@ -1,12 +1,16 @@import asynciofrom enum import Enum-from typing import List, Optional, Union+from typing import List, Optional, Tuple, Unionfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.metrics.utils import initialize_modelfrom deepteam.test_case import RTTestCase-from deepteam.attacks.attack_simulator.utils import generate, a_generate+from deepteam.attacks.attack_simulator.utils import (+ generate_with_cost,+ a_generate_with_cost,+ add_cost,+)from .template import AttackEngineTemplatesfrom .types import TransformedAttack, AttackVariations, ValidationResult@@ -47,7 +51,7 @@ class AttackEngine:vulnerability_type=vulnerability_type,generation_guidelines=self.generation_guidelines,)- transformed: TransformedAttack = generate(+ transformed, refine_cost = generate_with_cost(transform_prompt, TransformedAttack, self.simulator_model)transformed_input = transformed.input.strip()@@ -65,25 +69,37 @@ class AttackEngine:generation_guidelines=self.generation_guidelines,))- variations: AttackVariations = generate(+ variations, variation_cost = generate_with_cost(variation_prompt, AttackVariations, self.simulator_model)+ refine_cost = add_cost(refine_cost, variation_cost)candidates = [transformed_input] + [item.strip() for item in variations.inputs]- valid_inputs = self._validate_candidates_with_llm(+ valid_inputs, validation_cost = self._validate_candidates_with_llm(candidates=candidates,vulnerability=vulnerability,vulnerability_type=vulnerability_type,purpose=effective_purpose,)+ refine_cost = add_cost(refine_cost, validation_cost)if not valid_inputs:valid_inputs = [transformed_input]+ # Carry the baseline simulation_cost forward and add this refine+ # pass's cost, split evenly across the refined test cases produced.+ combined_cost = add_cost(test_case.simulation_cost, refine_cost)+ per_refined_cost = (+ combined_cost / len(valid_inputs)+ if combined_cost is not None and valid_inputs+ else combined_cost+ )refined_test_cases.extend([- self._clone_with_new_input(test_case, refined_input)+ self._clone_with_new_input(+ test_case, refined_input, per_refined_cost+ )for refined_input in valid_inputs])@@ -125,7 +141,7 @@ class AttackEngine:vulnerability_type=vulnerability_type,generation_guidelines=self.generation_guidelines,)- transformed: TransformedAttack = await a_generate(+ transformed, refine_cost = await a_generate_with_cost(transform_prompt, TransformedAttack, self.simulator_model)transformed_input = transformed.input.strip()@@ -143,24 +159,38 @@ class AttackEngine:generation_guidelines=self.generation_guidelines,))- variations: AttackVariations = await a_generate(+ variations, variation_cost = await a_generate_with_cost(variation_prompt, AttackVariations, self.simulator_model)+ refine_cost = add_cost(refine_cost, variation_cost)candidates = [transformed_input] + [item.strip() for item in variations.inputs]- valid_inputs = await self._a_validate_candidates_with_llm(- candidates=candidates,- vulnerability=vulnerability,- vulnerability_type=vulnerability_type,- purpose=effective_purpose,+ valid_inputs, validation_cost = (+ await self._a_validate_candidates_with_llm(+ candidates=candidates,+ vulnerability=vulnerability,+ vulnerability_type=vulnerability_type,+ purpose=effective_purpose,+ ))+ refine_cost = add_cost(refine_cost, validation_cost)if not valid_inputs:valid_inputs = [transformed_input]+ # Carry the baseline simulation_cost forward and add this refine pass's+ # cost, split evenly across the refined test cases produced.+ combined_cost = add_cost(test_case.simulation_cost, refine_cost)+ per_refined_cost = (+ combined_cost / len(valid_inputs)+ if combined_cost is not None and valid_inputs+ else combined_cost+ )return [- self._clone_with_new_input(test_case, refined_input)+ self._clone_with_new_input(+ test_case, refined_input, per_refined_cost+ )for refined_input in valid_inputs]@@ -180,9 +210,10 @@ class AttackEngine:vulnerability: str,vulnerability_type: Optional[str],purpose: Optional[str] = None,- ) -> List[str]:+ ) -> Tuple[List[str], Optional[float]]:effective_purpose = purpose if purpose is not None else self.purposevalidated: List[str] = []+ validation_cost = Nonefor candidate in candidates[: self.variations]:if not candidate or not candidate.strip():continue@@ -194,15 +225,16 @@ class AttackEngine:purpose=effective_purpose,)try:- result: ValidationResult = generate(+ result, cost = generate_with_cost(prompt, ValidationResult, self.simulator_model)+ validation_cost = add_cost(validation_cost, cost)if result.is_valid:validated.append(candidate)except Exception:continue- return validated+ return validated, validation_costasync def _a_validate_candidates_with_llm(self,@@ -210,13 +242,13 @@ class AttackEngine:vulnerability: str,vulnerability_type: Optional[str],purpose: Optional[str] = None,- ) -> List[str]:+ ) -> Tuple[List[str], Optional[float]]:effective_purpose = purpose if purpose is not None else self.purposetrimmed = [c for c in candidates[: self.variations] if c and c.strip()]if not trimmed:return []- async def validate_one(candidate: str) -> Optional[str]:+ async def validate_one(candidate: str):prompt = AttackEngineTemplates.validate_attack_template(candidate_input=candidate,vulnerability=vulnerability,@@ -224,19 +256,25 @@ class AttackEngine:purpose=effective_purpose,)try:- result: ValidationResult = await a_generate(+ result, cost = await a_generate_with_cost(prompt, ValidationResult, self.simulator_model)- return candidate if result.is_valid else None+ return (candidate if result.is_valid else None), costexcept Exception:- return None+ return None, Noneresults = await asyncio.gather(*[validate_one(c) for c in trimmed])- return [c for c in results if c]+ validated = [c for c, _ in results if c]+ validation_cost = None+ for _, cost in results:+ validation_cost = add_cost(validation_cost, cost)+ return validated, validation_cost@staticmethoddef _clone_with_new_input(- test_case: RTTestCase, refined_input: str+ test_case: RTTestCase,+ refined_input: str,+ simulation_cost: Optional[float] = None,) -> RTTestCase:return RTTestCase(vulnerability=test_case.vulnerability,@@ -244,4 +282,5 @@ class AttackEngine:input=refined_input,attack_method=test_case.attack_method,metadata=test_case.metadata,+ simulation_cost=simulation_cost,)diff --git a/deepteam/attacks/attack_simulator/attack_simulator.py b/deepteam/attacks/attack_simulator/attack_simulator.pyindex e6d2e73f..709d6707 100644--- a/deepteam/attacks/attack_simulator/attack_simulator.py+++ b/deepteam/attacks/attack_simulator/attack_simulator.py@@ -15,6 +15,10 @@ from deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.errors import ModelRefusalErrorfrom deepteam.test_case import RTTestCasefrom deepteam.attacks.attack_engine import AttackEngine+from deepteam.attacks.attack_simulator.utils import (+ cost_accumulator,+ add_cost,+)class BaselineAttack:@@ -284,17 +288,74 @@ class AttackSimulator:test_case.attack_method = attack.get_name()turns = [RTTurn(role="user", content=attack_input)]+ with cost_accumulator() as acc:+ try:+ res: List[RTTurn] = attack._get_turns(+ model_callback=self.model_callback,+ turns=turns,+ vulnerability=test_case.vulnerability,+ vulnerability_type=test_case.vulnerability_type.value,+ simulator_model=self.simulator_model,+ )++ test_case.turns = res+ test_case.actual_output = res[-1].content++ except ModelRefusalError as e:+ if ignore_errors:+ test_case.error = e.message+ return test_case+ else:+ raise+ except Exception as e:+ if ignore_errors:+ test_case.error = (+ f"Error enhancing multi-turn attack: {str(e)}"+ )+ return test_case+ else:+ raise+ finally:+ test_case.simulation_cost = add_cost(+ test_case.simulation_cost, acc[0]+ )++ return test_case++ # Regular attack handling (non-multi-turn)+ attack_input = test_case.input+ if attack_input is None:+ return test_case++ test_case.attack_method = attack.get_name()+ sig = inspect.signature(attack.enhance)++ with cost_accumulator() as acc:try:- res: List[RTTurn] = attack._get_turns(- model_callback=self.model_callback,- turns=turns,- vulnerability=test_case.vulnerability,- vulnerability_type=test_case.vulnerability_type.value,- simulator_model=self.simulator_model,- )+ res = None+ if (+ "simulator_model" in sig.parameters+ and "model_callback" in sig.parameters+ ):+ res = attack.enhance(+ attack=attack_input,+ simulator_model=self.simulator_model,+ model_callback=self.model_callback,+ )+ elif "simulator_model" in sig.parameters:+ res = attack.enhance(+ attack=attack_input,+ simulator_model=self.simulator_model,+ )+ elif "model_callback" in sig.parameters:+ res = attack.enhance(+ attack=attack_input,+ model_callback=self.model_callback,+ )+ else:+ res = attack.enhance(attack=attack_input)- test_case.turns = res- test_case.actual_output = res[-1].content+ test_case.input = resexcept ModelRefusalError as e:if ignore_errors:@@ -305,60 +366,15 @@ class AttackSimulator:except Exception as e:if ignore_errors:test_case.error = (- f"Error enhancing multi-turn attack: {str(e)}"+ f"Error enhancing regular attack: {str(e)}")return test_caseelse:raise-- return test_case-- # Regular attack handling (non-multi-turn)- attack_input = test_case.input- if attack_input is None:- return test_case-- test_case.attack_method = attack.get_name()- sig = inspect.signature(attack.enhance)-- try:- res = None- if (- "simulator_model" in sig.parameters- and "model_callback" in sig.parameters- ):- res = attack.enhance(- attack=attack_input,- simulator_model=self.simulator_model,- model_callback=self.model_callback,- )- elif "simulator_model" in sig.parameters:- res = attack.enhance(- attack=attack_input,- simulator_model=self.simulator_model,+ finally:+ test_case.simulation_cost = add_cost(+ test_case.simulation_cost, acc[0])- elif "model_callback" in sig.parameters:- res = attack.enhance(- attack=attack_input,- model_callback=self.model_callback,- )- else:- res = attack.enhance(attack=attack_input)-- test_case.input = res-- except ModelRefusalError as e:- if ignore_errors:- test_case.error = e.message- return test_case- else:- raise- except Exception as e:- if ignore_errors:- test_case.error = f"Error enhancing regular attack: {str(e)}"- return test_case- else:- raisereturn test_case@@ -383,17 +399,71 @@ class AttackSimulator:sig = inspect.signature(attack.a_enhance)turns = [RTTurn(role="user", content=attack_input)]+ with cost_accumulator() as acc:+ try:+ res: List[RTTurn] = await attack._a_get_turns(+ model_callback=self.model_callback,+ turns=turns,+ vulnerability=test_case.vulnerability,+ vulnerability_type=test_case.vulnerability_type.value,+ simulator_model=self.simulator_model,+ )++ test_case.turns = res+ test_case.actual_output = res[-1].content++ except ModelRefusalError as e:+ if ignore_errors:+ test_case.error = e.message+ return test_case+ else:+ raise+ except:+ if ignore_errors:+ test_case.error = "Error enhancing attack"+ return test_case+ else:+ raise+ finally:+ test_case.simulation_cost = add_cost(+ test_case.simulation_cost, acc[0]+ )++ return test_case++ attack_input = test_case.input+ if attack_input is None:+ return test_case++ test_case.attack_method = attack.get_name()+ sig = inspect.signature(attack.a_enhance)++ with cost_accumulator() as acc:try:- res: List[RTTurn] = await attack._a_get_turns(- model_callback=self.model_callback,- turns=turns,- vulnerability=test_case.vulnerability,- vulnerability_type=test_case.vulnerability_type.value,- simulator_model=self.simulator_model,- )+ res = None+ if (+ "simulator_model" in sig.parameters+ and "model_callback" in sig.parameters+ ):+ res = await attack.a_enhance(+ attack=attack_input,+ simulator_model=self.simulator_model,+ model_callback=self.model_callback,+ )+ elif "simulator_model" in sig.parameters:+ res = await attack.a_enhance(+ attack=attack_input,+ simulator_model=self.simulator_model,+ )+ elif "model_callback" in sig.parameters:+ res = await attack.a_enhance(+ attack=attack_input,+ model_callback=self.model_callback,+ )+ else:+ res = await attack.a_enhance(attack=attack_input)- test_case.turns = res- test_case.actual_output = res[-1].content+ test_case.input = resexcept ModelRefusalError as e:if ignore_errors:@@ -407,53 +477,9 @@ class AttackSimulator:return test_caseelse:raise-- return test_case-- attack_input = test_case.input- if attack_input is None:- return test_case-- test_case.attack_method = attack.get_name()- sig = inspect.signature(attack.a_enhance)-- try:- res = None- if (- "simulator_model" in sig.parameters- and "model_callback" in sig.parameters- ):- res = await attack.a_enhance(- attack=attack_input,- simulator_model=self.simulator_model,- model_callback=self.model_callback,- )- elif "simulator_model" in sig.parameters:- res = await attack.a_enhance(- attack=attack_input,- simulator_model=self.simulator_model,- )- elif "model_callback" in sig.parameters:- res = await attack.a_enhance(- attack=attack_input,- model_callback=self.model_callback,+ finally:+ test_case.simulation_cost = add_cost(+ test_case.simulation_cost, acc[0])- else:- res = await attack.a_enhance(attack=attack_input)-- test_case.input = res-- except ModelRefusalError as e:- if ignore_errors:- test_case.error = e.message- return test_case- else:- raise- except:- if ignore_errors:- test_case.error = "Error enhancing attack"- return test_case- else:- raisereturn test_casediff --git a/deepteam/attacks/attack_simulator/utils.py b/deepteam/attacks/attack_simulator/utils.pyindex cd5d2d22..0e546a28 100644--- a/deepteam/attacks/attack_simulator/utils.py+++ b/deepteam/attacks/attack_simulator/utils.py@@ -4,6 +4,8 @@ import osimport timeimport asyncioimport logging+from contextvars import ContextVar+from contextlib import contextmanagerfrom deepeval.metrics.utils import trimAndLoadJson, initialize_modelfrom deepeval.models import DeepEvalBaseLLMfrom deepteam.attacks.single_turn.escalation_constants import (@@ -14,13 +16,55 @@ from deepteam.attacks.single_turn.escalation_constants import (MAX_RETRIES = os.getenv("DEEPTEAM_MAX_RETRIES", 3)-def generate(+def add_cost(*costs):+ """Combine simulation/evaluation costs where ``None`` means 'no cost+ reported' (non-native models). Returns ``None`` only when every input is+ ``None``; otherwise treats ``None`` as ``0``.+ """+ if all(c is None for c in costs):+ return None+ return sum(c or 0 for c in costs)+++# Context-local simulation-cost accumulator. When a ``cost_accumulator()``+# scope is active, every ``generate``/``a_generate`` call records its cost into+# it. This lets callers (e.g. attack enhancement) capture the cost of an entire+# nested operation without threading cost through every method. ContextVars are+# concurrency-safe: each asyncio task gets its own copy, and child tasks created+# inside a scope share the same accumulator object.+_cost_acc: ContextVar = ContextVar("deepteam_simulation_cost_acc", default=None)+++def _record_cost(cost):+ """Add ``cost`` to the active accumulator, if any is in scope."""+ acc = _cost_acc.get()+ if acc is not None:+ acc[0] = add_cost(acc[0], cost)+++@contextmanager+def cost_accumulator():+ """Open a context-local accumulator that captures the simulation cost of all+ ``generate``/``a_generate`` calls made within the scope.++ Yields a single-element list whose ``[0]`` holds the running total (``None``+ until a native-model cost is recorded).+ """+ acc = [None]+ token = _cost_acc.set(acc)+ try:+ yield acc+ finally:+ _cost_acc.reset(token)+++def generate_with_cost(prompt: str,schema: BaseModel,model: DeepEvalBaseLLM = None,-) -> BaseModel:+):"""- Generate schema using the provided model with retry logic.+ Generate schema using the provided model with retry logic, returning cost.Args:prompt: The prompt to send to the model@@ -28,7 +72,9 @@ def generate(model: The model to useReturns:- The validated schema object+ A tuple ``(res, cost)`` where ``res`` is the validated schema object and+ ``cost`` is the model-reported cost for native models, or ``None`` for+ non-native models that do not report cost."""_, using_native_model = initialize_model(model=model)last_error = None@@ -37,34 +83,40 @@ def generate(for attempt in range(MAX_RETRIES):try:if using_native_model:- res, _ = model.generate(prompt=retry_prompt, schema=schema)+ res, cost = model.generate(prompt=retry_prompt, schema=schema)if res is None:- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")- return res+ raise ValueError(+ "Model returned None. This could be because of your model's guardrails, please use a different model."+ )+ return res, costelse:try:res = model.generate(prompt=retry_prompt, schema=schema)if res is None:- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")+ raise ValueError(+ "Model returned None. This could be because of your model's guardrails, please use a different model."+ )if isinstance(res, str):data = trimAndLoadJson(res)- return schema(**data)+ return schema(**data), Noneelse:- return res+ return res, Noneexcept TypeError:res = model.generate(retry_prompt)if res is None:- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")+ raise ValueError(+ "Model returned None. This could be because of your model's guardrails, please use a different model."+ )data = trimAndLoadJson(res)if schema == SyntheticDataList:data_list = [SyntheticData(**item) for item in data["data"]]- return SyntheticDataList(data=data_list)+ return SyntheticDataList(data=data_list), Noneelse:- return schema(**data)+ return schema(**data), Noneexcept Exception as e:last_error = e@@ -72,9 +124,7 @@ def generate(escalated_prompt = (f"{random_escalation_suffix(attempt)} \n\n {prompt}")- retry_prompt = append_critic_feedback(- escalated_prompt, str(e)- )+ retry_prompt = append_critic_feedback(escalated_prompt, str(e))sleep_time = 2**attemptlogging.warning(f"Generation failed on attempt {attempt + 1}. Retrying in {sleep_time}s... Error: {e}"@@ -86,13 +136,27 @@ def generate()-async def a_generate(+def generate(prompt: str,schema: BaseModel,- model: "DeepEvalBaseLLM" = None,+ model: DeepEvalBaseLLM = None,) -> BaseModel:+ """Backward-compatible wrapper around :func:`generate_with_cost` that+ returns only the validated schema object. Records cost into the active+ :func:`cost_accumulator` scope, if any."""+ res, cost = generate_with_cost(prompt, schema, model)+ _record_cost(cost)+ return res+++async def a_generate_with_cost(+ prompt: str,+ schema: BaseModel,+ model: "DeepEvalBaseLLM" = None,+):"""- Asynchronously generate schema using the provided model with retry logic.+ Asynchronously generate schema using the provided model with retry logic,+ returning cost.Args:prompt: The prompt to send to the model@@ -100,7 +164,9 @@ async def a_generate(model: The model to useReturns:- The validated schema object+ A tuple ``(res, cost)`` where ``res`` is the validated schema object and+ ``cost`` is the model-reported cost for native models, or ``None`` for+ non-native models that do not report cost."""_, using_native_model = initialize_model(model=model)last_error = None@@ -109,34 +175,44 @@ async def a_generate(for attempt in range(MAX_RETRIES):try:if using_native_model:- res, _ = await model.a_generate(prompt=retry_prompt, schema=schema)+ res, cost = await model.a_generate(+ prompt=retry_prompt, schema=schema+ )if res is None:- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")- return res+ raise ValueError(+ "Model returned None. This could be because of your model's guardrails, please use a different model."+ )+ return res, costelse:try:- res = await model.a_generate(prompt=retry_prompt, schema=schema)+ res = await model.a_generate(+ prompt=retry_prompt, schema=schema+ )if res is None:- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")+ raise ValueError(+ "Model returned None. This could be because of your model's guardrails, please use a different model."+ )if isinstance(res, str):data = trimAndLoadJson(res)- return schema(**data)+ return schema(**data), Noneelse:- return res+ return res, Noneexcept TypeError:res = await model.a_generate(retry_prompt)if res is None:- raise ValueError("Model returned None. This could be because of your model's guardrails, please use a different model.")+ raise ValueError(+ "Model returned None. This could be because of your model's guardrails, please use a different model."+ )data = trimAndLoadJson(res)if schema == SyntheticDataList:data_list = [SyntheticData(**item) for item in data["data"]]- return SyntheticDataList(data=data_list)+ return SyntheticDataList(data=data_list), Noneelse:- return schema(**data)+ return schema(**data), Noneexcept Exception as e:last_error = e@@ -144,9 +220,7 @@ async def a_generate(escalated_prompt = (f"{random_escalation_suffix(attempt)} \n\n {prompt}")- retry_prompt = append_critic_feedback(- escalated_prompt, str(e)- )+ retry_prompt = append_critic_feedback(escalated_prompt, str(e))sleep_time = 2**attemptlogging.warning(f"Async generation failed on attempt {attempt + 1}. Retrying in {sleep_time}s... Error: {e}"@@ -156,3 +230,16 @@ async def a_generate(raise RuntimeError(f"Failed to async generate after {MAX_RETRIES} attempts. Last error: {last_error}")+++async def a_generate(+ prompt: str,+ schema: BaseModel,+ model: "DeepEvalBaseLLM" = None,+) -> BaseModel:+ """Backward-compatible wrapper around :func:`a_generate_with_cost` that+ returns only the validated schema object. Records cost into the active+ :func:`cost_accumulator` scope, if any."""+ res, cost = await a_generate_with_cost(prompt, schema, model)+ _record_cost(cost)+ return res