DeepTeam is a framework to red team LLMs and AI agents.
pretty progress bars
A-Vamshi479c81869aef
diff --git a/deepteam/utils.py b/deepteam/utils.pyindex d2e6d184..b375d5a0 100644--- a/deepteam/utils.py+++ b/deepteam/utils.py@@ -6,7 +6,9 @@ from rich.progress import (SpinnerColumn,TextColumn,BarColumn,+ TaskProgressColumn,TimeElapsedColumn,+ TimeRemainingColumn,)from deepteam.test_case import RTTurn@@ -41,9 +43,13 @@ def format_turns(turns: List[RTTurn]):def create_progress() -> Progress:return Progress(SpinnerColumn(),- TextColumn("[progress.description]{task.description}"),- BarColumn(),+ TextColumn("[bold bright_white]{task.description}", justify="left"),+ BarColumn(bar_width=None),+ TaskProgressColumn(text_format="[cyan]{task.completed}[/]/[bright_white]{task.total}"),TimeElapsedColumn(),+ TimeRemainingColumn(compact=True),+ expand=True,+ transient=False,)diff --git a/test_something_dptm.py b/test_something_dptm.pynew file mode 100644index 00000000..8a14a938--- /dev/null+++ b/test_something_dptm.py@@ -0,0 +1,84 @@+from deepteam.attacks.multi_turn import (+ LinearJailbreaking,+ BadLikertJudge,+ CrescendoJailbreaking,+ SequentialJailbreak,+ TreeJailbreaking,+)+from deepteam.attacks.single_turn import (+ PromptInjection,+ Roleplay,+ GrayBox,+ Leetspeak,+ ROT13,+ Multilingual,+ MathProblem,+ Base64,+)+from deepteam import red_team+from deepteam.vulnerabilities import (+ Bias,+ RecursiveHijacking,+ Competition,+ BFLA,+ Ethics,+ Fairness,+ ChildProtection,+)+from openai import AsyncOpenAI, OpenAI+from deepteam.test_case.test_case import RTTurn, RTTestCase+++def model_callback(attack, turn_history=None):+ response = OpenAI().chat.completions.create(+ model="gpt-3.5-turbo",+ messages=[{"role": "user", "content": attack}],+ )+ return response.choices[0].message.content+++async def a_model_callback(attack, turn_history=None):+ response = await AsyncOpenAI().chat.completions.create(+ model="gpt-3.5-turbo",+ messages=[{"role": "user", "content": attack}],+ )+ return response.choices[0].message.content+++from deepteam.frameworks import Aegis, OWASPTop10, BeaverTails, NIST, MITRE+from deepteam.red_teamer import RedTeamer++beavertails = BeaverTails(num_attacks=5)++nist = NIST(categories=["measure_1", "measure_4"])+++if True:+ red_team(+ # vulnerabilities=[Bias(types=["gender"])],+ # attacks=[MathProblem()],+ framework=OWASPTop10(categories=["LLM_01", "LLM_02"]),+ model_callback=a_model_callback,+ simulator_model="gpt-4o-mini",+ evaluation_model="gpt-4o",+ )+else:+ red_team(+ # vulnerabilities=[Bias(types=["gender"])],+ # attacks=[MathProblem()],+ framework=NIST(),+ model_callback=model_callback,+ async_mode=False,+ simulator_model="gpt-4o-mini",+ evaluation_model="gpt-4o",+ )+++# teamer = RedTeamer(async_mode=False)++# res = teamer._framework_assessment(+# model_callback=model_callback,+# framework=nist+# )++# teamer._print_framework_overview_table(res)