Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

add progression folder to hold conversation state

A-Vamshid19e2f060d52
diff --git a/deepteam/attacks/multi_turn/progression/__init__.py b/deepteam/attacks/multi_turn/progression/__init__.py
new file mode 100644
index 00000000..f378eba4
--- /dev/null
+++ b/deepteam/attacks/multi_turn/progression/__init__.py
@@ -0,0 +1,41 @@
+from .progression import Progression, TURN_LEVEL_ATTACK_RATE
+from .types import Attempt, ProgressionResult
+from .stopping import (
+ StopReason,
+ default_stop_detail,
+ get_stopping_category,
+ get_stopping_reason,
+ is_progression_completed,
+ mark_stop,
+)
+from .shift_detection import (
+ AsyncMetricCheck,
+ BehaviorShiftDetector,
+ MetricCheck,
+ MetricCheckRecord,
+ MetricVerdict,
+ ShiftExplanation,
+ ShiftExplanationTemplate,
+ ShiftVerdict,
+)
+
+__all__ = [
+ "Progression",
+ "TURN_LEVEL_ATTACK_RATE",
+ "Attempt",
+ "ProgressionResult",
+ "StopReason",
+ "default_stop_detail",
+ "get_stopping_category",
+ "get_stopping_reason",
+ "is_progression_completed",
+ "mark_stop",
+ "AsyncMetricCheck",
+ "BehaviorShiftDetector",
+ "MetricCheck",
+ "MetricCheckRecord",
+ "MetricVerdict",
+ "ShiftExplanation",
+ "ShiftExplanationTemplate",
+ "ShiftVerdict",
+]
diff --git a/deepteam/attacks/multi_turn/progression/progression.py b/deepteam/attacks/multi_turn/progression/progression.py
new file mode 100644
index 00000000..bbae1418
--- /dev/null
+++ b/deepteam/attacks/multi_turn/progression/progression.py
@@ -0,0 +1,241 @@
+import random
+from typing import List, Optional, Union
+
+from deepeval.metrics.utils import initialize_model
+from deepeval.models import DeepEvalBaseLLM
+
+from deepteam.attacks.multi_turn.types import CallbackType
+from deepteam.attacks.multi_turn.utils import (
+ a_enhance_attack,
+ enhance_attack,
+)
+from deepteam.attacks.multi_turn.progression.shift_detection import (
+ AsyncMetricCheck,
+ BehaviorShiftDetector,
+ MetricCheck,
+ MetricVerdict,
+)
+from deepteam.attacks.multi_turn.progression.stopping import (
+ StopReason,
+ mark_stop,
+)
+from deepteam.attacks.multi_turn.progression.types import (
+ Attempt,
+ ProgressionResult,
+)
+from deepteam.attacks.single_turn import BaseSingleTurnAttack
+from deepteam.test_case.test_case import RTTurn
+
+TURN_LEVEL_ATTACK_RATE = 0.5
+
+
+class Progression:
+
+ def __init__(
+ self,
+ model_callback: CallbackType,
+ turns: Optional[List[RTTurn]] = None,
+ vulnerability: Optional[str] = None,
+ vulnerability_type: Optional[str] = None,
+ simulator_model: Optional[Union[str, DeepEvalBaseLLM]] = None,
+ *,
+ metric_check: Optional[Union[MetricCheck, AsyncMetricCheck]] = None,
+ turn_level_attacks: Optional[List[BaseSingleTurnAttack]] = None,
+ turn_level_attack_rate: float = TURN_LEVEL_ATTACK_RATE,
+ ):
+ self.model_callback = model_callback
+ self.turns: List[RTTurn] = list(turns or [])
+ self.simulator_model, _ = initialize_model(simulator_model)
+ self.turn_level_attacks = turn_level_attacks
+ self.turn_level_attack_rate = turn_level_attack_rate
+
+ self.goal = next(
+ (
+ turn.content
+ for turn in reversed(self.turns)
+ if turn.role == "user"
+ ),
+ None,
+ )
+ if self.goal is None:
+ raise ValueError(
+ "No user turn found in the conversation to progress from."
+ )
+
+ self.vulnerability_data = (
+ f"Vulnerability: {vulnerability} | Type: {vulnerability_type}"
+ )
+ self.detector = BehaviorShiftDetector(
+ simulator_model=self.simulator_model,
+ vulnerability_data=self.vulnerability_data,
+ metric_check=metric_check,
+ )
+
+ self.turns_spent = 0
+ self.attempts: List[Attempt] = []
+ # Attempt-tree bookkeeping. `_parent` is the attempt new probes
+ # descend from -- it follows commits and rolls back on a backtrack.
+ self._next_attempt_id = 0
+ self._parent: Optional[Attempt] = None
+ self._committed: List[Attempt] = []
+ self.stop_reason = StopReason.BUDGET_EXHAUSTED
+ self.stop_detail: Optional[str] = None
+ self.shift_verdict: Optional[MetricVerdict] = None
+ self.error: Optional[Exception] = None
+
+ @property
+ def last_response(self) -> str:
+ return self.turns[-1].content if self.turns else ""
+
+ def ensure_target_responded(self) -> None:
+ if len(self.turns) <= 1 or self.turns[-1].role == "user":
+ self.turns.append(self.model_callback(self.goal, self.turns))
+
+ async def a_ensure_target_responded(self) -> None:
+ if len(self.turns) <= 1 or self.turns[-1].role == "user":
+ self.turns.append(await self.model_callback(self.goal, self.turns))
+
+ def finalize(self) -> ProgressionResult:
+ mark_stop(
+ self.turns, self.stop_reason, self.stop_detail, self.turns_spent
+ )
+ return ProgressionResult(
+ turns=self.turns,
+ stop_reason=self.stop_reason,
+ stop_detail=self.stop_detail,
+ turns_spent=self.turns_spent,
+ attempts=self.attempts,
+ shift_verdict=self.shift_verdict,
+ error=self.error,
+ )
+
+ def probe(self, attack: str, *, enhance: bool = True) -> Attempt:
+ """Put an attack to the target without keeping it in the conversation.
+
+ For attacks that try several things per step and keep one: tree search
+ branches, retry-until-accepted loops, and knowledge-gathering questions
+ asked before the conversation proper begins. The Attempt is recorded
+ either way, so what was discarded survives in the result.
+ """
+ attack, turn_level_attack = self._apply_turn_level_attack(
+ attack, enhance
+ )
+ return self._record_attempt(
+ attack,
+ self.model_callback(attack, self.turns),
+ turn_level_attack,
+ )
+
+ async def a_probe(self, attack: str, *, enhance: bool = True) -> Attempt:
+ attack, turn_level_attack = await self._a_apply_turn_level_attack(
+ attack, enhance
+ )
+ return self._record_attempt(
+ attack,
+ await self.model_callback(attack, self.turns),
+ turn_level_attack,
+ )
+
+ def commit(self, attack: Union[str, Attempt]) -> RTTurn:
+ """Put an attack to the target and keep both turns.
+
+ Passing an Attempt from `probe` reuses the response already collected
+ rather than calling the target a second time.
+ """
+ attempt = attack if isinstance(attack, Attempt) else self.probe(attack)
+ return self._commit_attempt(attempt)
+
+ async def a_commit(self, attack: Union[str, Attempt]) -> RTTurn:
+ attempt = (
+ attack
+ if isinstance(attack, Attempt)
+ else await self.a_probe(attack)
+ )
+ return self._commit_attempt(attempt)
+
+ def shift_detected(self) -> bool:
+ """Has the target's behavior shifted? Records the stop reason when it
+ has, so the caller only has to return."""
+ return self._resolve_shift(self.detector.check(self.turns))
+
+ async def a_shift_detected(self) -> bool:
+ return self._resolve_shift(await self.detector.a_check(self.turns))
+
+ def remove_last_turns(self, count: int = 1) -> None:
+ """Drop the last `count` user/assistant pairs, for attacks that walk a
+ refused turn back and try a different approach."""
+ if count <= 0:
+ return
+ del self.turns[-2 * count :]
+ self.turns_spent = max(0, self.turns_spent - count)
+ # Walked-back turns are simply un-committed; the attempts stay in
+ # `attempts` with committed=False so the backtrack is reconstructable.
+ for _ in range(min(count, len(self._committed))):
+ self._committed.pop().committed = False
+ self._parent = self._committed[-1] if self._committed else None
+
+ def stop(self, reason: StopReason, detail: Optional[str] = None) -> None:
+ self.stop_reason = reason
+ self.stop_detail = detail
+
+ def _apply_turn_level_attack(self, attack: str, enhance: bool):
+ if not enhance or not self._should_apply_turn_level_attack():
+ return attack, None
+ picked = random.choice(self.turn_level_attacks)
+ return (
+ enhance_attack(picked, attack, self.simulator_model),
+ picked.get_name(),
+ )
+
+ async def _a_apply_turn_level_attack(self, attack: str, enhance: bool):
+ if not enhance or not self._should_apply_turn_level_attack():
+ return attack, None
+ picked = random.choice(self.turn_level_attacks)
+ return (
+ await a_enhance_attack(picked, attack, self.simulator_model),
+ picked.get_name(),
+ )
+
+ def _should_apply_turn_level_attack(self) -> bool:
+ return bool(self.turn_level_attacks) and (
+ random.random() < self.turn_level_attack_rate
+ )
+
+ def _record_attempt(
+ self,
+ attack: str,
+ response: RTTurn,
+ turn_level_attack: Optional[str],
+ ) -> Attempt:
+ attempt = Attempt(
+ id=self._next_attempt_id,
+ parent_id=self._parent.id if self._parent else None,
+ depth=self._parent.depth + 1 if self._parent else 0,
+ attack=attack,
+ response=response,
+ turn_level_attack=turn_level_attack,
+ )
+ self._next_attempt_id += 1
+ self.attempts.append(attempt)
+ return attempt
+
+ def _commit_attempt(self, attempt: Attempt) -> RTTurn:
+ attempt.committed = True
+ attempt.response.turn_level_attack = attempt.turn_level_attack
+ self.turns.append(RTTurn(role="user", content=attempt.attack))
+ self.turns.append(attempt.response)
+ self.turns_spent += 1
+ self._committed.append(attempt)
+ self._parent = attempt
+ return attempt.response
+
+ def _resolve_shift(self, verdict) -> bool:
+ if verdict is None:
+ return False
+ self.shift_verdict = MetricVerdict(
+ score=verdict.metric_score,
+ reason=verdict.metric_reason,
+ evaluation_cost=verdict.evaluation_cost,
+ )
+ self.stop(StopReason.SHIFT_DETECTED, verdict.detail)
+ return True
diff --git a/deepteam/attacks/multi_turn/progression.py b/deepteam/attacks/multi_turn/progression/shift_detection.py
similarity index 63%
rename from deepteam/attacks/multi_turn/progression.py
rename to deepteam/attacks/multi_turn/progression/shift_detection.py
index f54e28b2..b0306497 100644
--- a/deepteam/attacks/multi_turn/progression.py
+++ b/deepteam/attacks/multi_turn/progression/shift_detection.py
@@ -1,5 +1,4 @@
import inspect
-from enum import Enum
from typing import Awaitable, Callable, List, Optional, Union
from deepeval.models import DeepEvalBaseLLM
@@ -14,30 +13,6 @@ from deepteam.test_case.test_case import RTTurn
from deepteam.utils import format_turns
-class StopReason(str, Enum):
- SHIFT_DETECTED = "SHIFT_DETECTED"
- BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED"
- SIMULATOR_REFUSED = "SIMULATOR_REFUSED"
- SIMULATION_ERROR = "SIMULATION_ERROR"
- RUNTIME_EXCEEDED = "RUNTIME_EXCEEDED"
-
-
-_INCOMPLETE_STOP_REASONS = frozenset(
- {
- StopReason.SIMULATOR_REFUSED,
- StopReason.SIMULATION_ERROR,
- StopReason.RUNTIME_EXCEEDED,
- }
-)
-
-
-def is_progression_completed(reason: Optional[Union[StopReason, str]]) -> bool:
- if reason is None:
- return True
- value = reason.value if isinstance(reason, StopReason) else reason
- return value not in {r.value for r in _INCOMPLETE_STOP_REASONS}
-
-
def _as_sentence(text: str) -> str:
text = " ".join(text.split())
if text and text[-1] not in ".!?":
@@ -45,88 +20,6 @@ def _as_sentence(text: str) -> str:
return text
-def default_stop_detail(
- reason: StopReason, turns_spent: Optional[int] = None
-) -> str:
- if reason is StopReason.SHIFT_DETECTED:
- return (
- "The target's behavior shifted on this turn, ending the "
- "conversation early."
- )
- if reason is StopReason.BUDGET_EXHAUSTED:
- if turns_spent:
- return (
- f"The target held for its full budget of {turns_spent} "
- "turn(s). No behavioral shift was detected at any turn."
- )
- return (
- "The target held for its full turn budget. No behavioral shift "
- "was detected at any turn."
- )
- if reason is StopReason.SIMULATOR_REFUSED:
- return (
- "The simulator model refused to write the next attack, so the "
- "conversation ended before the turn budget was spent. This says "
- "nothing about how long the target resisted."
- )
- if reason is StopReason.SIMULATION_ERROR:
- return (
- "The attack progression failed before the turn budget was spent, "
- "so this conversation's length does not reflect the target's "
- "resistance."
- )
- return (
- "The attack hit its runtime backstop before exhausting its turn "
- "budget, so this conversation's length does not reflect the "
- "target's resistance."
- )
-
-
-def mark_stop(
- turns: List[RTTurn],
- reason: Union[StopReason, str],
- detail: Optional[str] = None,
- turns_spent: Optional[int] = None,
-) -> List[RTTurn]:
- if not turns:
- return turns
-
- if isinstance(reason, StopReason):
- category = reason
- else:
- try:
- category = StopReason(reason)
- except ValueError:
- category = None
-
- turns[-1].stopping_category = (
- category.value if category is not None else str(reason)
- )
- if detail:
- turns[-1].stopping_reason = detail
- elif category is not None:
- turns[-1].stopping_reason = default_stop_detail(category, turns_spent)
- else:
- turns[-1].stopping_reason = str(reason)
- return turns
-
-
-def get_stopping_category(turns: Optional[List[RTTurn]]) -> Optional[str]:
- if not turns:
- return None
- return turns[-1].stopping_category
-
-
-def get_stopping_reason(turns: Optional[List[RTTurn]]) -> Optional[str]:
- if not turns:
- return None
- return turns[-1].stopping_reason
-
-
-class ShiftExplanation(BaseModel):
- reasoning: str
-
-
class MetricVerdict(BaseModel):
score: Optional[float] = None
reason: Optional[str] = None
@@ -155,6 +48,10 @@ AsyncMetricCheck = Callable[
]
+class ShiftExplanation(BaseModel):
+ reasoning: str
+
+
class ShiftVerdict(BaseModel):
reasoning: Optional[str] = None
metric_score: Optional[float] = None
@@ -216,9 +113,7 @@ class BehaviorShiftDetector:
def _can_check(self, turns: Optional[List[RTTurn]]) -> bool:
if self.metric_check is None:
return False
- return bool(turns) and any(
- turn.role == "assistant" for turn in turns
- )
+ return bool(turns) and any(turn.role == "assistant" for turn in turns)
def _explain(self, turns: List[RTTurn]) -> Optional[str]:
try:
diff --git a/deepteam/attacks/multi_turn/progression/stopping.py b/deepteam/attacks/multi_turn/progression/stopping.py
new file mode 100644
index 00000000..443da2bb
--- /dev/null
+++ b/deepteam/attacks/multi_turn/progression/stopping.py
@@ -0,0 +1,108 @@
+from enum import Enum
+from typing import List, Optional, Union
+
+from deepteam.test_case.test_case import RTTurn
+
+
+class StopReason(str, Enum):
+ SHIFT_DETECTED = "SHIFT_DETECTED"
+ BUDGET_EXHAUSTED = "BUDGET_EXHAUSTED"
+ SIMULATOR_REFUSED = "SIMULATOR_REFUSED"
+ SIMULATION_ERROR = "SIMULATION_ERROR"
+ RUNTIME_EXCEEDED = "RUNTIME_EXCEEDED"
+
+
+# Reasons that mean the run was cut short by our side, not by the target holding
+# out — a conversation that ends this way says nothing about the target.
+_INCOMPLETE_STOP_REASONS = frozenset(
+ {
+ StopReason.SIMULATOR_REFUSED,
+ StopReason.SIMULATION_ERROR,
+ StopReason.RUNTIME_EXCEEDED,
+ }
+)
+
+
+def is_progression_completed(reason: Optional[Union[StopReason, str]]) -> bool:
+ if reason is None:
+ return True
+ value = reason.value if isinstance(reason, StopReason) else reason
+ return value not in {r.value for r in _INCOMPLETE_STOP_REASONS}
+
+
+def default_stop_detail(
+ reason: StopReason, turns_spent: Optional[int] = None
+) -> str:
+ if reason is StopReason.SHIFT_DETECTED:
+ return (
+ "The target's behavior shifted on this turn, ending the "
+ "conversation early."
+ )
+ if reason is StopReason.BUDGET_EXHAUSTED:
+ if turns_spent:
+ return (
+ f"The target held for its full budget of {turns_spent} "
+ "turn(s). No behavioral shift was detected at any turn."
+ )
+ return (
+ "The target held for its full turn budget. No behavioral shift "
+ "was detected at any turn."
+ )
+ if reason is StopReason.SIMULATOR_REFUSED:
+ return (
+ "The simulator model refused to write the next attack, so the "
+ "conversation ended before the turn budget was spent. This says "
+ "nothing about how long the target resisted."
+ )
+ if reason is StopReason.SIMULATION_ERROR:
+ return (
+ "The attack progression failed before the turn budget was spent, "
+ "so this conversation's length does not reflect the target's "
+ "resistance."
+ )
+ return (
+ "The attack hit its runtime backstop before exhausting its turn "
+ "budget, so this conversation's length does not reflect the "
+ "target's resistance."
+ )
+
+
+def mark_stop(
+ turns: List[RTTurn],
+ reason: Union[StopReason, str],
+ detail: Optional[str] = None,
+ turns_spent: Optional[int] = None,
+) -> List[RTTurn]:
+ if not turns:
+ return turns
+
+ if isinstance(reason, StopReason):
+ category = reason
+ else:
+ try:
+ category = StopReason(reason)
+ except ValueError:
+ category = None
+
+ turns[-1].stopping_category = (
+ category.value if category is not None else str(reason)
+ )
+ if detail:
+ turns[-1].stopping_reason = detail
+ elif category is not None:
+ turns[-1].stopping_reason = default_stop_detail(category, turns_spent)
+ else:
+ turns[-1].stopping_reason = str(reason)
+ return turns
+
+
+def get_stopping_category(turns: Optional[List[RTTurn]]) -> Optional[str]:
+ if not turns:
+ return None
+ return turns[-1].stopping_category
+
+
+def get_stopping_reason(turns: Optional[List[RTTurn]]) -> Optional[str]:
+ if not turns:
+ return None
+ return turns[-1].stopping_reason
diff --git a/deepteam/attacks/multi_turn/progression/types.py b/deepteam/attacks/multi_turn/progression/types.py
new file mode 100644
index 00000000..7eb8d279
--- /dev/null
+++ b/deepteam/attacks/multi_turn/progression/types.py
@@ -0,0 +1,35 @@
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict
+
+from deepteam.attacks.multi_turn.progression.shift_detection import (
+ MetricVerdict,
+)
+from deepteam.attacks.multi_turn.progression.stopping import StopReason
+from deepteam.test_case.test_case import RTTurn
+
+
+class Attempt(BaseModel):
+ id: int
+ parent_id: Optional[int] = None
+ attack: str
+ response: RTTurn
+ turn_level_attack: Optional[str] = None
+ committed: bool = False
+ depth: int = 0
+
+
+class ProgressionResult(BaseModel):
+ model_config = ConfigDict(arbitrary_types_allowed=True)
+
+ turns: List[RTTurn]
+ stop_reason: StopReason
+ stop_detail: Optional[str] = None
+ turns_spent: int = 0
+ attempts: List[Attempt] = []
+ shift_verdict: Optional[MetricVerdict] = None
+ error: Optional[Exception] = None
+
+ @property
+ def last_response(self) -> Optional[str]:
+ return self.turns[-1].content if self.turns else None