Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

pretty progress bars

A-Vamshi479c81869aef
diff --git a/deepteam/utils.py b/deepteam/utils.py
index d2e6d184..b375d5a0 100644
--- a/deepteam/utils.py
+++ b/deepteam/utils.py
@@ -6,7 +6,9 @@ from rich.progress import (
SpinnerColumn,
TextColumn,
BarColumn,
+ TaskProgressColumn,
TimeElapsedColumn,
+ TimeRemainingColumn,
)
from deepteam.test_case import RTTurn
@@ -41,9 +43,13 @@ def format_turns(turns: List[RTTurn]):
def create_progress() -> Progress:
return Progress(
SpinnerColumn(),
- TextColumn("[progress.description]{task.description}"),
- BarColumn(),
+ TextColumn("[bold bright_white]{task.description}", justify="left"),
+ BarColumn(bar_width=None),
+ TaskProgressColumn(text_format="[cyan]{task.completed}[/]/[bright_white]{task.total}"),
TimeElapsedColumn(),
+ TimeRemainingColumn(compact=True),
+ expand=True,
+ transient=False,
)
diff --git a/test_something_dptm.py b/test_something_dptm.py
new file mode 100644
index 00000000..8a14a938
--- /dev/null
+++ b/test_something_dptm.py
@@ -0,0 +1,84 @@
+from deepteam.attacks.multi_turn import (
+ LinearJailbreaking,
+ BadLikertJudge,
+ CrescendoJailbreaking,
+ SequentialJailbreak,
+ TreeJailbreaking,
+)
+from deepteam.attacks.single_turn import (
+ PromptInjection,
+ Roleplay,
+ GrayBox,
+ Leetspeak,
+ ROT13,
+ Multilingual,
+ MathProblem,
+ Base64,
+)
+from deepteam import red_team
+from deepteam.vulnerabilities import (
+ Bias,
+ RecursiveHijacking,
+ Competition,
+ BFLA,
+ Ethics,
+ Fairness,
+ ChildProtection,
+)
+from openai import AsyncOpenAI, OpenAI
+from deepteam.test_case.test_case import RTTurn, RTTestCase
+
+
+def model_callback(attack, turn_history=None):
+ response = OpenAI().chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": attack}],
+ )
+ return response.choices[0].message.content
+
+
+async def a_model_callback(attack, turn_history=None):
+ response = await AsyncOpenAI().chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": attack}],
+ )
+ return response.choices[0].message.content
+
+
+from deepteam.frameworks import Aegis, OWASPTop10, BeaverTails, NIST, MITRE
+from deepteam.red_teamer import RedTeamer
+
+beavertails = BeaverTails(num_attacks=5)
+
+nist = NIST(categories=["measure_1", "measure_4"])
+
+
+if True:
+ red_team(
+ # vulnerabilities=[Bias(types=["gender"])],
+ # attacks=[MathProblem()],
+ framework=OWASPTop10(categories=["LLM_01", "LLM_02"]),
+ model_callback=a_model_callback,
+ simulator_model="gpt-4o-mini",
+ evaluation_model="gpt-4o",
+ )
+else:
+ red_team(
+ # vulnerabilities=[Bias(types=["gender"])],
+ # attacks=[MathProblem()],
+ framework=NIST(),
+ model_callback=model_callback,
+ async_mode=False,
+ simulator_model="gpt-4o-mini",
+ evaluation_model="gpt-4o",
+ )
+
+
+# teamer = RedTeamer(async_mode=False)
+
+# res = teamer._framework_assessment(
+# model_callback=model_callback,
+# framework=nist
+# )
+
+# teamer._print_framework_overview_table(res)