DeepTeam is a framework to red team LLMs and AI agents.
update tests
A-Vamshi55faa9473fd0
diff --git a/tests/test_core/test_attacks/test_bad_likert_judge.py b/tests/test_core/test_attacks/test_bad_likert_judge.pyindex f3cfc200..23820dd6 100644--- a/tests/test_core/test_attacks/test_bad_likert_judge.py+++ b/tests/test_core/test_attacks/test_bad_likert_judge.py@@ -1,6 +1,7 @@import pytestfrom unittest.mock import AsyncMock, MagicMockfrom deepteam.attacks.multi_turn import BadLikertJudge+from deepteam.attacks.multi_turn.progression import ProgressionResultfrom deepteam.vulnerabilities import Biasfrom deepteam.test_case.test_case import RTTurnfrom deepteam.red_teamer.utils import wrap_model_callback@@ -53,11 +54,14 @@ class TestBadLikertJudge:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestBadLikertJudge.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestBadLikertJudge.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestBadLikertJudge.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):@@ -98,11 +102,14 @@ class TestBadLikertJudge:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestBadLikertJudge.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestBadLikertJudge.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestBadLikertJudge.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):diff --git a/tests/test_core/test_attacks/test_early_stopping.py b/tests/test_core/test_attacks/test_early_stopping.pyindex 6a2ca8da..fe9410d6 100644--- a/tests/test_core/test_attacks/test_early_stopping.py+++ b/tests/test_core/test_attacks/test_early_stopping.py@@ -23,7 +23,8 @@ from deepteam.attacks.multi_turn.progression import (from deepteam.test_case.test_case import RTTurnMODULES = [- "deepteam.attacks.multi_turn.progression",+ "deepteam.attacks.multi_turn.progression.shift_detection",+ "deepteam.attacks.multi_turn.progression.progression","deepteam.attacks.multi_turn.linear_jailbreaking.linear_jailbreaking","deepteam.attacks.multi_turn.bad_likert_judge.bad_likert_judge","deepteam.attacks.multi_turn.sequential_break.sequential_break",@@ -112,7 +113,7 @@ class TestBehaviorShiftDetector:@pytest.fixture(autouse=True)def _stub_explainer(self, monkeypatch):- import deepteam.attacks.multi_turn.progression as progression+ import deepteam.attacks.multi_turn.progression.shift_detection as progressionmonkeypatch.setattr(progression,@@ -154,7 +155,7 @@ class TestBehaviorShiftDetector:assert detector.check([RTTurn(role="user", content="attack")]) is Nonedef test_explainer_failure_still_stops(self, monkeypatch):- import deepteam.attacks.multi_turn.progression as progression+ import deepteam.attacks.multi_turn.progression.shift_detection as progressiondef boom(*a, **k):raise RuntimeError("explainer down")@@ -378,7 +379,7 @@ class TestCrescendoBacktracking:def test_refusals_cost_budget_but_never_end_the_run(self, monkeypatch):attack = CrescendoJailbreaking(max_rounds=3, max_backtracks=2)monkeypatch.setattr(- attack, "get_refusal_score", lambda *a, **k: (True, "refused")+ attack, "_refusal_score", lambda *a, **k: (True, "refused"))turns = attack._get_turns(@@ -393,7 +394,7 @@ class TestCrescendoBacktracking:attack = CrescendoJailbreaking(max_rounds=2, max_backtracks=1)monkeypatch.setattr(attack,- "get_refusal_score",+ "_refusal_score",lambda *a, **k: (True, "secret-rationale"),)@@ -412,17 +413,17 @@ class TestTreeSearch:def test_transcript_uses_the_scored_responses(self):attack = TreeJailbreaking(max_depth=2, branching_factor=2)- turns = attack._get_turns(+ result = attack.run(make_callback(), seed(), metric_check=lambda t: PASS)- committed = [n for n in attack.search_log if n["committed"]]+ committed = [a for a in result.attempts if a.committed]assert len(committed) == 2- assert [n.prompt for n in attack.committed_path] == [- t.content for t in turns if t.role == "user"+ assert [a.attack for a in committed] == [+ t.content for t in result.turns if t.role == "user"][1:]- for node in attack.committed_path:- assert node.response is not None+ for attempt in committed:+ assert attempt.response is not Nonedef test_depth_bounds_the_path_not_the_clock(self):turns = TreeJailbreaking(max_depth=4, branching_factor=1)._get_turns(@@ -456,8 +457,8 @@ class TestTreeSearch:monkeypatch.setattr(tree, "generate", all_off_topic)attack = TreeJailbreaking(max_depth=2, branching_factor=2)- turns = attack._get_turns(+ result = attack.run(make_callback(), seed(), metric_check=lambda t: PASS)- assert len(turns) == 2 + 2 * 2- assert not any(n["on_topic"] for n in attack.search_log)+ assert len(result.turns) == 2 + 2 * 2+ assert not any(a.on_topic for a in result.attempts)diff --git a/tests/test_core/test_attacks/test_linear_jailbreaking.py b/tests/test_core/test_attacks/test_linear_jailbreaking.pyindex e95fdcac..a3c96ed7 100644--- a/tests/test_core/test_attacks/test_linear_jailbreaking.py+++ b/tests/test_core/test_attacks/test_linear_jailbreaking.py@@ -1,6 +1,7 @@import pytestfrom unittest.mock import AsyncMock, MagicMockfrom deepteam.attacks.multi_turn import LinearJailbreaking+from deepteam.attacks.multi_turn.progression import ProgressionResultfrom deepteam.vulnerabilities import Biasfrom deepteam.test_case.test_case import RTTurnfrom deepteam.red_teamer.utils import wrap_model_callback@@ -53,11 +54,14 @@ class TestLinearJailbreaking:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestLinearJailbreaking.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):@@ -99,11 +103,14 @@ class TestLinearJailbreaking:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestLinearJailbreaking.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestLinearJailbreaking.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):diff --git a/tests/test_core/test_attacks/test_sequential_jailbreak.py b/tests/test_core/test_attacks/test_sequential_jailbreak.pyindex 1ef7f925..9eaa492a 100644--- a/tests/test_core/test_attacks/test_sequential_jailbreak.py+++ b/tests/test_core/test_attacks/test_sequential_jailbreak.py@@ -2,6 +2,7 @@ import pytestfrom unittest.mock import AsyncMock, MagicMockfrom deepteam.attacks.multi_turn import SequentialJailbreakfrom deepteam.test_case.test_case import RTTurn+from deepteam.attacks.multi_turn.progression import ProgressionResultfrom deepteam.vulnerabilities import Biasfrom deepteam.red_teamer.utils import wrap_model_callback@@ -53,11 +54,14 @@ class TestSequentialJailbreak:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestSequentialJailbreak.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):@@ -98,11 +102,14 @@ class TestSequentialJailbreak:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestSequentialJailbreak.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestSequentialJailbreak.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):diff --git a/tests/test_core/test_attacks/test_tree_jailbreaking.py b/tests/test_core/test_attacks/test_tree_jailbreaking.pyindex 6388b040..ab6af98a 100644--- a/tests/test_core/test_attacks/test_tree_jailbreaking.py+++ b/tests/test_core/test_attacks/test_tree_jailbreaking.py@@ -2,6 +2,7 @@ import pytestfrom unittest.mock import AsyncMock, MagicMockfrom deepteam.attacks.multi_turn import TreeJailbreakingfrom deepteam.test_case.test_case import RTTurn+from deepteam.attacks.multi_turn.progression import ProgressionResultfrom deepteam.vulnerabilities import Biasfrom deepteam.red_teamer.utils import wrap_model_callback@@ -53,11 +54,14 @@ class TestTreeJailbreaking:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestTreeJailbreaking.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):@@ -98,11 +102,14 @@ class TestTreeJailbreaking:)assert len(type_to_turns.keys()) == 1- assert isinstance(- type_to_turns.get(TestTreeJailbreaking.BIAS.types[0]), list- ) and all(+ results = type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])+ assert isinstance(results, list) and all(+ isinstance(result, ProgressionResult) for result in results+ )+ assert all(isinstance(turn, RTTurn)- for turn in type_to_turns.get(TestTreeJailbreaking.BIAS.types[0])+ for result in results+ for turn in result.turns)assert user_only_turns_result[1].role == "assistant"with pytest.raises(ValueError):