Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

update tests

A-Vamshi55faa9473fd0
diff --git a/tests/test_core/test_attacks/test_bad_likert_judge.py b/tests/test_core/test_attacks/test_bad_likert_judge.py
index f3cfc200..23820dd6 100644
--- a/tests/test_core/test_attacks/test_bad_likert_judge.py
+++ b/tests/test_core/test_attacks/test_bad_likert_judge.py
@@ -1,6 +1,7 @@
import pytest
from unittest.mock import AsyncMock, MagicMock
from deepteam.attacks.multi_turn import BadLikertJudge
+from deepteam.attacks.multi_turn.progression import ProgressionResult
from deepteam.vulnerabilities import Bias
from deepteam.test_case.test_case import RTTurn
from deepteam.red_teamer.utils import wrap_model_callback
@@ -53,11 +54,14 @@ class TestBadLikertJudge:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestBadLikertJudge.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestBadLikertJudge.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestBadLikertJudge.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
@@ -98,11 +102,14 @@ class TestBadLikertJudge:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestBadLikertJudge.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestBadLikertJudge.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestBadLikertJudge.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
diff --git a/tests/test_core/test_attacks/test_early_stopping.py b/tests/test_core/test_attacks/test_early_stopping.py
index 6a2ca8da..fe9410d6 100644
--- a/tests/test_core/test_attacks/test_early_stopping.py
+++ b/tests/test_core/test_attacks/test_early_stopping.py
@@ -23,7 +23,8 @@ from deepteam.attacks.multi_turn.progression import (
from deepteam.test_case.test_case import RTTurn
MODULES = [
- "deepteam.attacks.multi_turn.progression",
+ "deepteam.attacks.multi_turn.progression.shift_detection",
+ "deepteam.attacks.multi_turn.progression.progression",
"deepteam.attacks.multi_turn.linear_jailbreaking.linear_jailbreaking",
"deepteam.attacks.multi_turn.bad_likert_judge.bad_likert_judge",
"deepteam.attacks.multi_turn.sequential_break.sequential_break",
@@ -112,7 +113,7 @@ class TestBehaviorShiftDetector:
@pytest.fixture(autouse=True)
def _stub_explainer(self, monkeypatch):
- import deepteam.attacks.multi_turn.progression as progression
+ import deepteam.attacks.multi_turn.progression.shift_detection as progression
monkeypatch.setattr(
progression,
@@ -154,7 +155,7 @@ class TestBehaviorShiftDetector:
assert detector.check([RTTurn(role="user", content="attack")]) is None
def test_explainer_failure_still_stops(self, monkeypatch):
- import deepteam.attacks.multi_turn.progression as progression
+ import deepteam.attacks.multi_turn.progression.shift_detection as progression
def boom(*a, **k):
raise RuntimeError("explainer down")
@@ -378,7 +379,7 @@ class TestCrescendoBacktracking:
def test_refusals_cost_budget_but_never_end_the_run(self, monkeypatch):
attack = CrescendoJailbreaking(max_rounds=3, max_backtracks=2)
monkeypatch.setattr(
- attack, "get_refusal_score", lambda *a, **k: (True, "refused")
+ attack, "_refusal_score", lambda *a, **k: (True, "refused")
)
turns = attack._get_turns(
@@ -393,7 +394,7 @@ class TestCrescendoBacktracking:
attack = CrescendoJailbreaking(max_rounds=2, max_backtracks=1)
monkeypatch.setattr(
attack,
- "get_refusal_score",
+ "_refusal_score",
lambda *a, **k: (True, "secret-rationale"),
)
@@ -412,17 +413,17 @@ class TestTreeSearch:
def test_transcript_uses_the_scored_responses(self):
attack = TreeJailbreaking(max_depth=2, branching_factor=2)
- turns = attack._get_turns(
+ result = attack.run(
make_callback(), seed(), metric_check=lambda t: PASS
)
- committed = [n for n in attack.search_log if n["committed"]]
+ committed = [a for a in result.attempts if a.committed]
assert len(committed) == 2
- assert [n.prompt for n in attack.committed_path] == [
- t.content for t in turns if t.role == "user"
+ assert [a.attack for a in committed] == [
+ t.content for t in result.turns if t.role == "user"
][1:]
- for node in attack.committed_path:
- assert node.response is not None
+ for attempt in committed:
+ assert attempt.response is not None
def test_depth_bounds_the_path_not_the_clock(self):
turns = TreeJailbreaking(max_depth=4, branching_factor=1)._get_turns(
@@ -456,8 +457,8 @@ class TestTreeSearch:
monkeypatch.setattr(tree, "generate", all_off_topic)
attack = TreeJailbreaking(max_depth=2, branching_factor=2)
- turns = attack._get_turns(
+ result = attack.run(
make_callback(), seed(), metric_check=lambda t: PASS
)
- assert len(turns) == 2 + 2 * 2
- assert not any(n["on_topic"] for n in attack.search_log)
+ assert len(result.turns) == 2 + 2 * 2
+ assert not any(a.on_topic for a in result.attempts)
diff --git a/tests/test_core/test_attacks/test_linear_jailbreaking.py b/tests/test_core/test_attacks/test_linear_jailbreaking.py
index e95fdcac..a3c96ed7 100644
--- a/tests/test_core/test_attacks/test_linear_jailbreaking.py
+++ b/tests/test_core/test_attacks/test_linear_jailbreaking.py
@@ -1,6 +1,7 @@
import pytest
from unittest.mock import AsyncMock, MagicMock
from deepteam.attacks.multi_turn import LinearJailbreaking
+from deepteam.attacks.multi_turn.progression import ProgressionResult
from deepteam.vulnerabilities import Bias
from deepteam.test_case.test_case import RTTurn
from deepteam.red_teamer.utils import wrap_model_callback
@@ -53,11 +54,14 @@ class TestLinearJailbreaking:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestLinearJailbreaking.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
@@ -99,11 +103,14 @@ class TestLinearJailbreaking:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestLinearJailbreaking.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
diff --git a/tests/test_core/test_attacks/test_sequential_jailbreak.py b/tests/test_core/test_attacks/test_sequential_jailbreak.py
index 1ef7f925..9eaa492a 100644
--- a/tests/test_core/test_attacks/test_sequential_jailbreak.py
+++ b/tests/test_core/test_attacks/test_sequential_jailbreak.py
@@ -2,6 +2,7 @@ import pytest
from unittest.mock import AsyncMock, MagicMock
from deepteam.attacks.multi_turn import SequentialJailbreak
from deepteam.test_case.test_case import RTTurn
+from deepteam.attacks.multi_turn.progression import ProgressionResult
from deepteam.vulnerabilities import Bias
from deepteam.red_teamer.utils import wrap_model_callback
@@ -53,11 +54,14 @@ class TestSequentialJailbreak:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestSequentialJailbreak.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
@@ -98,11 +102,14 @@ class TestSequentialJailbreak:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestSequentialJailbreak.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
diff --git a/tests/test_core/test_attacks/test_tree_jailbreaking.py b/tests/test_core/test_attacks/test_tree_jailbreaking.py
index 6388b040..ab6af98a 100644
--- a/tests/test_core/test_attacks/test_tree_jailbreaking.py
+++ b/tests/test_core/test_attacks/test_tree_jailbreaking.py
@@ -2,6 +2,7 @@ import pytest
from unittest.mock import AsyncMock, MagicMock
from deepteam.attacks.multi_turn import TreeJailbreaking
from deepteam.test_case.test_case import RTTurn
+from deepteam.attacks.multi_turn.progression import ProgressionResult
from deepteam.vulnerabilities import Bias
from deepteam.red_teamer.utils import wrap_model_callback
@@ -53,11 +54,14 @@ class TestTreeJailbreaking:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestTreeJailbreaking.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):
@@ -98,11 +102,14 @@ class TestTreeJailbreaking:
)
assert len(type_to_turns.keys()) == 1
- assert isinstance(
- type_to_turns.get(TestTreeJailbreaking.BIAS.types[0]), list
- ) and all(
+ results = type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])
+ assert isinstance(results, list) and all(
+ isinstance(result, ProgressionResult) for result in results
+ )
+ assert all(
isinstance(turn, RTTurn)
- for turn in type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])
+ for result in results
+ for turn in result.turns
)
assert user_only_turns_result[1].role == "assistant"
with pytest.raises(ValueError):