DeepTeam is a framework to red team LLMs and AI agents.
add progression folder to hold conversation state
A-Vamshid19e2f060d52
diff --git a/deepteam/attacks/multi_turn/progression/__init__.py b/deepteam/attacks/multi_turn/progression/__init__.pynew file mode 100644index 00000000..f378eba4--- /dev/null+++ b/deepteam/attacks/multi_turn/progression/__init__.py@@ -0,0 +1,41 @@+from .progression import Progression, TURN_LEVEL_ATTACK_RATE+from .types import Attempt, ProgressionResult+from .stopping import (+ StopReason,+ default_stop_detail,+ get_stopping_category,+ get_stopping_reason,+ is_progression_completed,+ mark_stop,+)+from .shift_detection import (+ AsyncMetricCheck,+ BehaviorShiftDetector,+ MetricCheck,+ MetricCheckRecord,+ MetricVerdict,+ ShiftExplanation,+ ShiftExplanationTemplate,+ ShiftVerdict,+)++__all__ = [+ "Progression",+ "TURN_LEVEL_ATTACK_RATE",+ "Attempt",+ "ProgressionResult",+ "StopReason",+ "default_stop_detail",+ "get_stopping_category",+ "get_stopping_reason",+ "is_progression_completed",+ "mark_stop",+ "AsyncMetricCheck",+ "BehaviorShiftDetector",+ "MetricCheck",+ "MetricCheckRecord",+ "MetricVerdict",+ "ShiftExplanation",+ "ShiftExplanationTemplate",+ "ShiftVerdict",+]diff --git a/deepteam/attacks/multi_turn/progression/progression.py b/deepteam/attacks/multi_turn/progression/progression.pynew file mode 100644index 00000000..bbae1418--- /dev/null+++ b/deepteam/attacks/multi_turn/progression/progression.py@@ -0,0 +1,241 @@+import random+from typing import List, Optional, Union++from deepeval.metrics.utils import initialize_model+from deepeval.models import DeepEvalBaseLLM++from deepteam.attacks.multi_turn.types import CallbackType+from deepteam.attacks.multi_turn.utils import (+ a_enhance_attack,+ enhance_attack,+)+from deepteam.attacks.multi_turn.progression.shift_detection import (+ AsyncMetricCheck,+ BehaviorShiftDetector,+ MetricCheck,+ MetricVerdict,+)+from deepteam.attacks.multi_turn.progression.stopping import (+ StopReason,+ mark_stop,+)+from deepteam.attacks.multi_turn.progression.types import (+ Attempt,+ ProgressionResult,+)+from deepteam.attacks.single_turn import BaseSingleTurnAttack+from deepteam.test_case.test_case import RTTurn++TURN_LEVEL_ATTACK_RATE = 0.5+++class Progression:++ def __init__(+ self,+ model_callback: CallbackType,+ turns: Optional[List[RTTurn]] = None,+ vulnerability: Optional[str] = None,+ vulnerability_type: Optional[str] = None,+ simulator_model: Optional[Union[str, DeepEvalBaseLLM]] = None,+ *,+ metric_check: Optional[Union[MetricCheck, AsyncMetricCheck]] = None,+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,+ turn_level_attack_rate: float = TURN_LEVEL_ATTACK_RATE,+ ):+ self.model_callback = model_callback+ self.turns: List[RTTurn] = list(turns or [])+ self.simulator_model, _ = initialize_model(simulator_model)+ self.turn_level_attacks = turn_level_attacks+ self.turn_level_attack_rate = turn_level_attack_rate++ self.goal = next(+ (+ turn.content+ for turn in reversed(self.turns)+ if turn.role == "user"+ ),+ None,+ )+ if self.goal is None:+ raise ValueError(+ "No user turn found in the conversation to progress from."+ )++ self.vulnerability_data = (+ f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"+ )+ self.detector = BehaviorShiftDetector(+ simulator_model=self.simulator_model,+ vulnerability_data=self.vulnerability_data,+ metric_check=metric_check,+ )++ self.turns_spent = 0+ self.attempts: List[Attempt] = []+ # Attempt-tree bookkeeping. `_parent` is the attempt new probes+ # descend from -- it follows commits and rolls back on a backtrack.+ self._next_attempt_id = 0+ self._parent: Optional[Attempt] = None+ self._committed: List[Attempt] = []+ self.stop_reason = StopReason.BUDGET_EXHAUSTED+ self.stop_detail: Optional[str] = None+ self.shift_verdict: Optional[MetricVerdict] = None+ self.error: Optional[Exception] = None++ @property+ def last_response(self) -> str:+ return self.turns[-1].content if self.turns else ""++ def ensure_target_responded(self) -> None:+ if len(self.turns) <= 1 or self.turns[-1].role == "user":+ self.turns.append(self.model_callback(self.goal, self.turns))++ async def a_ensure_target_responded(self) -> None:+ if len(self.turns) <= 1 or self.turns[-1].role == "user":+ self.turns.append(await self.model_callback(self.goal, self.turns))++ def finalize(self) -> ProgressionResult:+ mark_stop(+ self.turns, self.stop_reason, self.stop_detail, self.turns_spent+ )+ return ProgressionResult(+ turns=self.turns,+ stop_reason=self.stop_reason,+ stop_detail=self.stop_detail,+ turns_spent=self.turns_spent,+ attempts=self.attempts,+ shift_verdict=self.shift_verdict,+ error=self.error,+ )++ def probe(self, attack: str, *, enhance: bool = True) -> Attempt:+ """Put an attack to the target without keeping it in the conversation.++ For attacks that try several things per step and keep one: tree search+ branches, retry-until-accepted loops, and knowledge-gathering questions+ asked before the conversation proper begins. The Attempt is recorded+ either way, so what was discarded survives in the result.+ """+ attack, turn_level_attack = self._apply_turn_level_attack(+ attack, enhance+ )+ return self._record_attempt(+ attack,+ self.model_callback(attack, self.turns),+ turn_level_attack,+ )++ async def a_probe(self, attack: str, *, enhance: bool = True) -> Attempt:+ attack, turn_level_attack = await self._a_apply_turn_level_attack(+ attack, enhance+ )+ return self._record_attempt(+ attack,+ await self.model_callback(attack, self.turns),+ turn_level_attack,+ )++ def commit(self, attack: Union[str, Attempt]) -> RTTurn:+ """Put an attack to the target and keep both turns.++ Passing an Attempt from `probe` reuses the response already collected+ rather than calling the target a second time.+ """+ attempt = attack if isinstance(attack, Attempt) else self.probe(attack)+ return self._commit_attempt(attempt)++ async def a_commit(self, attack: Union[str, Attempt]) -> RTTurn:+ attempt = (+ attack+ if isinstance(attack, Attempt)+ else await self.a_probe(attack)+ )+ return self._commit_attempt(attempt)++ def shift_detected(self) -> bool:+ """Has the target's behavior shifted? Records the stop reason when it+ has, so the caller only has to return."""+ return self._resolve_shift(self.detector.check(self.turns))++ async def a_shift_detected(self) -> bool:+ return self._resolve_shift(await self.detector.a_check(self.turns))++ def remove_last_turns(self, count: int = 1) -> None:+ """Drop the last `count` user/assistant pairs, for attacks that walk a+ refused turn back and try a different approach."""+ if count <= 0:+ return+ del self.turns[-2 * count :]+ self.turns_spent = max(0, self.turns_spent - count)+ # Walked-back turns are simply un-committed; the attempts stay in+ # `attempts` with committed=False so the backtrack is reconstructable.+ for _ in range(min(count, len(self._committed))):+ self._committed.pop().committed = False+ self._parent = self._committed[-1] if self._committed else None++ def stop(self, reason: StopReason, detail: Optional[str] = None) -> None:+ self.stop_reason = reason+ self.stop_detail = detail++ def _apply_turn_level_attack(self, attack: str, enhance: bool):+ if not enhance or not self._should_apply_turn_level_attack():+ return attack, None+ picked = random.choice(self.turn_level_attacks)+ return (+ enhance_attack(picked, attack, self.simulator_model),+ picked.get_name(),+ )++ async def _a_apply_turn_level_attack(self, attack: str, enhance: bool):+ if not enhance or not self._should_apply_turn_level_attack():+ return attack, None+ picked = random.choice(self.turn_level_attacks)+ return (+ await a_enhance_attack(picked, attack, self.simulator_model),+ picked.get_name(),+ )++ def _should_apply_turn_level_attack(self) -> bool:+ return bool(self.turn_level_attacks) and (+ random.random() < self.turn_level_attack_rate+ )++ def _record_attempt(+ self,+ attack: str,+ response: RTTurn,+ turn_level_attack: Optional[str],+ ) -> Attempt:+ attempt = Attempt(+ id=self._next_attempt_id,+ parent_id=self._parent.id if self._parent else None,+ depth=self._parent.depth + 1 if self._parent else 0,+ attack=attack,+ response=response,+ turn_level_attack=turn_level_attack,+ )+ self._next_attempt_id += 1+ self.attempts.append(attempt)+ return attempt++ def _commit_attempt(self, attempt: Attempt) -> RTTurn:+ attempt.committed = True+ attempt.response.turn_level_attack = attempt.turn_level_attack+ self.turns.append(RTTurn(role="user", content=attempt.attack))+ self.turns.append(attempt.response)+ self.turns_spent += 1+ self._committed.append(attempt)+ self._parent = attempt+ return attempt.response++ def _resolve_shift(self, verdict) -> bool:+ if verdict is None:+ return False+ self.shift_verdict = MetricVerdict(+ score=verdict.metric_score,+ reason=verdict.metric_reason,+ evaluation_cost=verdict.evaluation_cost,+ )+ self.stop(StopReason.SHIFT_DETECTED, verdict.detail)+ return Truediff --git a/deepteam/attacks/multi_turn/progression.py b/deepteam/attacks/multi_turn/progression/shift_detection.pysimilarity index 63%rename from deepteam/attacks/multi_turn/progression.pyrename to deepteam/attacks/multi_turn/progression/shift_detection.pyindex f54e28b2..b0306497 100644--- a/deepteam/attacks/multi_turn/progression.py+++ b/deepteam/attacks/multi_turn/progression/shift_detection.py@@ -1,5 +1,4 @@import inspect-from enum import Enumfrom typing import Awaitable, Callable, List, Optional, Unionfrom deepeval.models import DeepEvalBaseLLM@@ -14,30 +13,6 @@ from deepteam.test_case.test_case import RTTurnfrom deepteam.utils import format_turns-class StopReason(str, Enum):- SHIFT_DETECTED = "SHIFT_DETECTED"- BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED"- SIMULATOR_REFUSED = "SIMULATOR_REFUSED"- SIMULATION_ERROR = "SIMULATION_ERROR"- RUNTIME_EXCEEDED = "RUNTIME_EXCEEDED"---_INCOMPLETE_STOP_REASONS = frozenset(- {- StopReason.SIMULATOR_REFUSED,- StopReason.SIMULATION_ERROR,- StopReason.RUNTIME_EXCEEDED,- }-)---def is_progression_completed(reason: Optional[Union[StopReason, str]]) -> bool:- if reason is None:- return True- value = reason.value if isinstance(reason, StopReason) else reason- return value not in {r.value for r in _INCOMPLETE_STOP_REASONS}--def _as_sentence(text: str) -> str:text = " ".join(text.split())if text and text[-1] not in ".!?":@@ -45,88 +20,6 @@ def _as_sentence(text: str) -> str:return text-def default_stop_detail(- reason: StopReason, turns_spent: Optional[int] = None-) -> str:- if reason is StopReason.SHIFT_DETECTED:- return (- "The target's behavior shifted on this turn, ending the "- "conversation early."- )- if reason is StopReason.BUDGET_EXHAUSTED:- if turns_spent:- return (- f"The target held for its full budget of {turns_spent} "- "turn(s). No behavioral shift was detected at any turn."- )- return (- "The target held for its full turn budget. No behavioral shift "- "was detected at any turn."- )- if reason is StopReason.SIMULATOR_REFUSED:- return (- "The simulator model refused to write the next attack, so the "- "conversation ended before the turn budget was spent. This says "- "nothing about how long the target resisted."- )- if reason is StopReason.SIMULATION_ERROR:- return (- "The attack progression failed before the turn budget was spent, "- "so this conversation's length does not reflect the target's "- "resistance."- )- return (- "The attack hit its runtime backstop before exhausting its turn "- "budget, so this conversation's length does not reflect the "- "target's resistance."- )---def mark_stop(- turns: List[RTTurn],- reason: Union[StopReason, str],- detail: Optional[str] = None,- turns_spent: Optional[int] = None,-) -> List[RTTurn]:- if not turns:- return turns-- if isinstance(reason, StopReason):- category = reason- else:- try:- category = StopReason(reason)- except ValueError:- category = None-- turns[-1].stopping_category = (- category.value if category is not None else str(reason)- )- if detail:- turns[-1].stopping_reason = detail- elif category is not None:- turns[-1].stopping_reason = default_stop_detail(category, turns_spent)- else:- turns[-1].stopping_reason = str(reason)- return turns---def get_stopping_category(turns: Optional[List[RTTurn]]) -> Optional[str]:- if not turns:- return None- return turns[-1].stopping_category---def get_stopping_reason(turns: Optional[List[RTTurn]]) -> Optional[str]:- if not turns:- return None- return turns[-1].stopping_reason---class ShiftExplanation(BaseModel):- reasoning: str--class MetricVerdict(BaseModel):score: Optional[float] = Nonereason: Optional[str] = None@@ -155,6 +48,10 @@ AsyncMetricCheck = Callable[]+class ShiftExplanation(BaseModel):+ reasoning: str++class ShiftVerdict(BaseModel):reasoning: Optional[str] = Nonemetric_score: Optional[float] = None@@ -216,9 +113,7 @@ class BehaviorShiftDetector:def _can_check(self, turns: Optional[List[RTTurn]]) -> bool:if self.metric_check is None:return False- return bool(turns) and any(- turn.role == "assistant" for turn in turns- )+ return bool(turns) and any(turn.role == "assistant" for turn in turns)def _explain(self, turns: List[RTTurn]) -> Optional[str]:try:diff --git a/deepteam/attacks/multi_turn/progression/stopping.py b/deepteam/attacks/multi_turn/progression/stopping.pynew file mode 100644index 00000000..443da2bb--- /dev/null+++ b/deepteam/attacks/multi_turn/progression/stopping.py@@ -0,0 +1,108 @@+from enum import Enum+from typing import List, Optional, Union++from deepteam.test_case.test_case import RTTurn+++class StopReason(str, Enum):+ SHIFT_DETECTED = "SHIFT_DETECTED"+ BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED"+ SIMULATOR_REFUSED = "SIMULATOR_REFUSED"+ SIMULATION_ERROR = "SIMULATION_ERROR"+ RUNTIME_EXCEEDED = "RUNTIME_EXCEEDED"+++# Reasons that mean the run was cut short by our side, not by the target holding+# out — a conversation that ends this way says nothing about the target.+_INCOMPLETE_STOP_REASONS = frozenset(+ {+ StopReason.SIMULATOR_REFUSED,+ StopReason.SIMULATION_ERROR,+ StopReason.RUNTIME_EXCEEDED,+ }+)+++def is_progression_completed(reason: Optional[Union[StopReason, str]]) -> bool:+ if reason is None:+ return True+ value = reason.value if isinstance(reason, StopReason) else reason+ return value not in {r.value for r in _INCOMPLETE_STOP_REASONS}+++def default_stop_detail(+ reason: StopReason, turns_spent: Optional[int] = None+) -> str:+ if reason is StopReason.SHIFT_DETECTED:+ return (+ "The target's behavior shifted on this turn, ending the "+ "conversation early."+ )+ if reason is StopReason.BUDGET_EXHAUSTED:+ if turns_spent:+ return (+ f"The target held for its full budget of {turns_spent} "+ "turn(s). No behavioral shift was detected at any turn."+ )+ return (+ "The target held for its full turn budget. No behavioral shift "+ "was detected at any turn."+ )+ if reason is StopReason.SIMULATOR_REFUSED:+ return (+ "The simulator model refused to write the next attack, so the "+ "conversation ended before the turn budget was spent. This says "+ "nothing about how long the target resisted."+ )+ if reason is StopReason.SIMULATION_ERROR:+ return (+ "The attack progression failed before the turn budget was spent, "+ "so this conversation's length does not reflect the target's "+ "resistance."+ )+ return (+ "The attack hit its runtime backstop before exhausting its turn "+ "budget, so this conversation's length does not reflect the "+ "target's resistance."+ )+++def mark_stop(+ turns: List[RTTurn],+ reason: Union[StopReason, str],+ detail: Optional[str] = None,+ turns_spent: Optional[int] = None,+) -> List[RTTurn]:+ if not turns:+ return turns++ if isinstance(reason, StopReason):+ category = reason+ else:+ try:+ category = StopReason(reason)+ except ValueError:+ category = None++ turns[-1].stopping_category = (+ category.value if category is not None else str(reason)+ )+ if detail:+ turns[-1].stopping_reason = detail+ elif category is not None:+ turns[-1].stopping_reason = default_stop_detail(category, turns_spent)+ else:+ turns[-1].stopping_reason = str(reason)+ return turns+++def get_stopping_category(turns: Optional[List[RTTurn]]) -> Optional[str]:+ if not turns:+ return None+ return turns[-1].stopping_category+++def get_stopping_reason(turns: Optional[List[RTTurn]]) -> Optional[str]:+ if not turns:+ return None+ return turns[-1].stopping_reasondiff --git a/deepteam/attacks/multi_turn/progression/types.py b/deepteam/attacks/multi_turn/progression/types.pynew file mode 100644index 00000000..7eb8d279--- /dev/null+++ b/deepteam/attacks/multi_turn/progression/types.py@@ -0,0 +1,35 @@+from typing import List, Optional++from pydantic import BaseModel, ConfigDict++from deepteam.attacks.multi_turn.progression.shift_detection import (+ MetricVerdict,+)+from deepteam.attacks.multi_turn.progression.stopping import StopReason+from deepteam.test_case.test_case import RTTurn+++class Attempt(BaseModel):+ id: int+ parent_id: Optional[int] = None+ attack: str+ response: RTTurn+ turn_level_attack: Optional[str] = None+ committed: bool = False+ depth: int = 0+++class ProgressionResult(BaseModel):+ model_config = ConfigDict(arbitrary_types_allowed=True)++ turns: List[RTTurn]+ stop_reason: StopReason+ stop_detail: Optional[str] = None+ turns_spent: int = 0+ attempts: List[Attempt] = []+ shift_verdict: Optional[MetricVerdict] = None+ error: Optional[Exception] = None++ @property+ def last_response(self) -> Optional[str]:+ return self.turns[-1].content if self.turns else None