DeepTeam is a framework to red team LLMs and AI agents.
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"authorship_tag": "ABX9TyNwIDQIlsu+ZzpWul+ZsKbM",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/github/A-Vamshi/deepteam/blob/main/examples/custom_red_teaming.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "RmW-833YVU2R",
"outputId": "7e7b9ad3-39d8-4e28-fd0b-4f626958ce6d"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting deepteam\n",
" Downloading deepteam-0.2.7-py3-none-any.whl.metadata (15 kB)\n",
"Collecting deepeval\n",
" Downloading deepeval-3.6.7-py3-none-any.whl.metadata (18 kB)\n",
"Requirement already satisfied: aiohttp>=3.11.18 in /usr/local/lib/python3.12/dist-packages (from deepteam) (3.13.0)\n",
" Downloading deepeval-3.6.2-py3-none-any.whl.metadata (18 kB)\n",
"Requirement already satisfied: grpcio>=1.67.1 in /usr/local/lib/python3.12/dist-packages (from deepteam) (1.75.1)\n",
"Requirement already satisfied: openai>=1.76.2 in /usr/local/lib/python3.12/dist-packages (from deepteam) (1.109.1)\n",
"Requirement already satisfied: pyyaml>=6.0.2 in /usr/local/lib/python3.12/dist-packages (from deepteam) (6.0.3)\n",
"Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.12/dist-packages (from deepteam) (2.32.4)\n",
"Requirement already satisfied: tabulate>=0.9.0 in /usr/local/lib/python3.12/dist-packages (from deepteam) (0.9.0)\n",
"Requirement already satisfied: tqdm>=4.66.1 in /usr/local/lib/python3.12/dist-packages (from deepteam) (4.67.1)\n",
"Collecting anthropic (from deepeval)\n",
" Downloading anthropic-0.71.0-py3-none-any.whl.metadata (28 kB)\n",
"Collecting click<8.3.0,>=8.0.0 (from deepeval)\n",
" Downloading click-8.2.1-py3-none-any.whl.metadata (2.5 kB)\n",
"Requirement already satisfied: google-genai<2.0.0,>=1.9.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.41.0)\n",
"Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from deepeval) (3.1.6)\n",
"Requirement already satisfied: nest_asyncio in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.6.0)\n",
"Collecting ollama (from deepeval)\n",
" Downloading ollama-0.6.0-py3-none-any.whl.metadata (4.3 kB)\n",
"Requirement already satisfied: opentelemetry-api<2.0.0,>=1.24.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.37.0)\n",
"Collecting opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0 (from deepeval)\n",
" Downloading opentelemetry_exporter_otlp_proto_grpc-1.38.0-py3-none-any.whl.metadata (2.4 kB)\n",
"Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.24.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.37.0)\n",
"Collecting portalocker (from deepeval)\n",
" Downloading portalocker-3.2.0-py3-none-any.whl.metadata (8.7 kB)\n",
"Collecting posthog<7.0.0,>=6.3.0 (from deepeval)\n",
" Downloading posthog-6.7.8-py3-none-any.whl.metadata (6.0 kB)\n",
"Requirement already satisfied: pydantic<3.0.0,>=2.11.7 in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.11.10)\n",
"Requirement already satisfied: pydantic-settings<3.0.0,>=2.10.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.11.0)\n",
"Collecting pyfiglet (from deepeval)\n",
" Downloading pyfiglet-1.0.4-py3-none-any.whl.metadata (7.4 kB)\n",
"Requirement already satisfied: pytest in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.4.2)\n",
"Collecting pytest-asyncio (from deepeval)\n",
" Downloading pytest_asyncio-1.2.0-py3-none-any.whl.metadata (4.1 kB)\n",
"Collecting pytest-repeat (from deepeval)\n",
" Downloading pytest_repeat-0.9.4-py3-none-any.whl.metadata (4.9 kB)\n",
"Collecting pytest-rerunfailures<13.0,>=12.0 (from deepeval)\n",
" Downloading pytest_rerunfailures-12.0-py3-none-any.whl.metadata (18 kB)\n",
"Collecting pytest-xdist (from deepeval)\n",
" Downloading pytest_xdist-3.8.0-py3-none-any.whl.metadata (3.0 kB)\n",
"Requirement already satisfied: python-dotenv<2.0.0,>=1.1.1 in /usr/local/lib/python3.12/dist-packages (from deepeval) (1.1.1)\n",
"Requirement already satisfied: rich<15.0.0,>=13.6.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (13.9.4)\n",
"Requirement already satisfied: sentry-sdk in /usr/local/lib/python3.12/dist-packages (from deepeval) (2.40.0)\n",
"Requirement already satisfied: setuptools in /usr/local/lib/python3.12/dist-packages (from deepeval) (75.2.0)\n",
"Requirement already satisfied: tenacity<=10.0.0,>=8.0.0 in /usr/local/lib/python3.12/dist-packages (from deepeval) (8.5.0)\n",
"Requirement already satisfied: typer<1.0.0,>=0.9 in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.19.2)\n",
"Requirement already satisfied: wheel in /usr/local/lib/python3.12/dist-packages (from deepeval) (0.45.1)\n",
"Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (2.6.1)\n",
"Requirement already satisfied: aiosignal>=1.4.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (1.4.0)\n",
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (25.4.0)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (1.8.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (6.7.0)\n",
"Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (0.3.2)\n",
"Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.12/dist-packages (from aiohttp>=3.11.18->deepteam) (1.22.0)\n",
"Requirement already satisfied: anyio<5.0.0,>=4.8.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (4.11.0)\n",
"Requirement already satisfied: google-auth<3.0.0,>=2.14.1 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (2.38.0)\n",
"Requirement already satisfied: httpx<1.0.0,>=0.28.1 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (0.28.1)\n",
"Requirement already satisfied: websockets<15.1.0,>=13.0.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (15.0.1)\n",
"Requirement already satisfied: typing-extensions<5.0.0,>=4.11.0 in /usr/local/lib/python3.12/dist-packages (from google-genai<2.0.0,>=1.9.0->deepeval) (4.15.0)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.12/dist-packages (from openai>=1.76.2->deepteam) (1.9.0)\n",
"Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from openai>=1.76.2->deepteam) (0.11.0)\n",
"Requirement already satisfied: sniffio in /usr/local/lib/python3.12/dist-packages (from openai>=1.76.2->deepteam) (1.3.1)\n",
"Requirement already satisfied: importlib-metadata<8.8.0,>=6.0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-api<2.0.0,>=1.24.0->deepeval) (8.7.0)\n",
"Requirement already satisfied: googleapis-common-protos~=1.57 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (1.70.0)\n",
"Collecting opentelemetry-exporter-otlp-proto-common==1.38.0 (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval)\n",
" Downloading opentelemetry_exporter_otlp_proto_common-1.38.0-py3-none-any.whl.metadata (1.8 kB)\n",
"Collecting opentelemetry-proto==1.38.0 (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval)\n",
" Downloading opentelemetry_proto-1.38.0-py3-none-any.whl.metadata (2.3 kB)\n",
"Collecting opentelemetry-sdk<2.0.0,>=1.24.0 (from deepeval)\n",
" Downloading opentelemetry_sdk-1.38.0-py3-none-any.whl.metadata (1.5 kB)\n",
"Requirement already satisfied: protobuf<7.0,>=5.0 in /usr/local/lib/python3.12/dist-packages (from opentelemetry-proto==1.38.0->opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (5.29.5)\n",
"Collecting opentelemetry-api<2.0.0,>=1.24.0 (from deepeval)\n",
" Downloading opentelemetry_api-1.38.0-py3-none-any.whl.metadata (1.5 kB)\n",
"Collecting opentelemetry-semantic-conventions==0.59b0 (from opentelemetry-sdk<2.0.0,>=1.24.0->deepeval)\n",
" Downloading opentelemetry_semantic_conventions-0.59b0-py3-none-any.whl.metadata (2.4 kB)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (1.17.0)\n",
"Requirement already satisfied: python-dateutil>=2.2 in /usr/local/lib/python3.12/dist-packages (from posthog<7.0.0,>=6.3.0->deepeval) (2.9.0.post0)\n",
"Collecting backoff>=1.10.0 (from posthog<7.0.0,>=6.3.0->deepeval)\n",
" Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.11.7->deepeval) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.33.2 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.11.7->deepeval) (2.33.2)\n",
"Requirement already satisfied: typing-inspection>=0.4.0 in /usr/local/lib/python3.12/dist-packages (from pydantic<3.0.0,>=2.11.7->deepeval) (0.4.2)\n",
"Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.12/dist-packages (from pytest-rerunfailures<13.0,>=12.0->deepeval) (25.0)\n",
"Requirement already satisfied: iniconfig>=1 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (2.1.0)\n",
"Requirement already satisfied: pluggy<2,>=1.5 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (1.6.0)\n",
"Requirement already satisfied: pygments>=2.7.2 in /usr/local/lib/python3.12/dist-packages (from pytest->deepeval) (2.19.2)\n",
"Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (3.4.3)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (3.10)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (2.5.0)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.12/dist-packages (from requests>=2.31.0->deepteam) (2025.10.5)\n",
"Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich<15.0.0,>=13.6.0->deepeval) (4.0.0)\n",
"Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer<1.0.0,>=0.9->deepeval) (1.5.4)\n",
"Requirement already satisfied: docstring-parser<1,>=0.15 in /usr/local/lib/python3.12/dist-packages (from anthropic->deepeval) (0.17.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->deepeval) (3.0.3)\n",
"Collecting execnet>=2.1 (from pytest-xdist->deepeval)\n",
" Downloading execnet-2.1.1-py3-none-any.whl.metadata (2.9 kB)\n",
"Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (5.5.2)\n",
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.4.2)\n",
"Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.12/dist-packages (from google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (4.9.1)\n",
"Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx<1.0.0,>=0.28.1->google-genai<2.0.0,>=1.9.0->deepeval) (1.0.9)\n",
"Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx<1.0.0,>=0.28.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.16.0)\n",
"Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.12/dist-packages (from importlib-metadata<8.8.0,>=6.0->opentelemetry-api<2.0.0,>=1.24.0->deepeval) (3.23.0)\n",
"Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich<15.0.0,>=13.6.0->deepeval) (0.1.2)\n",
"Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /usr/local/lib/python3.12/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.0,>=2.14.1->google-genai<2.0.0,>=1.9.0->deepeval) (0.6.1)\n",
"Downloading deepteam-0.2.7-py3-none-any.whl (459 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m459.1/459.1 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading deepeval-3.6.2-py3-none-any.whl (617 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m617.7/617.7 kB\u001b[0m \u001b[31m24.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading click-8.2.1-py3-none-any.whl (102 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.2/102.2 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_grpc-1.38.0-py3-none-any.whl (19 kB)\n",
"Downloading opentelemetry_exporter_otlp_proto_common-1.38.0-py3-none-any.whl (18 kB)\n",
"Downloading opentelemetry_proto-1.38.0-py3-none-any.whl (72 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.5/72.5 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading opentelemetry_sdk-1.38.0-py3-none-any.whl (132 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.3/132.3 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading opentelemetry_api-1.38.0-py3-none-any.whl (65 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.9/65.9 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading opentelemetry_semantic_conventions-0.59b0-py3-none-any.whl (207 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m208.0/208.0 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading posthog-6.7.8-py3-none-any.whl (137 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.3/137.3 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading pytest_rerunfailures-12.0-py3-none-any.whl (12 kB)\n",
"Downloading anthropic-0.71.0-py3-none-any.whl (355 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m355.0/355.0 kB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading ollama-0.6.0-py3-none-any.whl (14 kB)\n",
"Downloading portalocker-3.2.0-py3-none-any.whl (22 kB)\n",
"Downloading pyfiglet-1.0.4-py3-none-any.whl (1.8 MB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m50.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading pytest_asyncio-1.2.0-py3-none-any.whl (15 kB)\n",
"Downloading pytest_repeat-0.9.4-py3-none-any.whl (4.2 kB)\n",
"Downloading pytest_xdist-3.8.0-py3-none-any.whl (46 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.4/46.4 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
"Downloading execnet-2.1.1-py3-none-any.whl (40 kB)\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.6/40.6 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hInstalling collected packages: pyfiglet, portalocker, opentelemetry-proto, execnet, click, backoff, pytest-xdist, pytest-rerunfailures, pytest-repeat, pytest-asyncio, posthog, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, opentelemetry-semantic-conventions, ollama, anthropic, opentelemetry-sdk, opentelemetry-exporter-otlp-proto-grpc, deepeval, deepteam\n",
" Attempting uninstall: click\n",
" Found existing installation: click 8.3.0\n",
" Uninstalling click-8.3.0:\n",
" Successfully uninstalled click-8.3.0\n",
" Attempting uninstall: opentelemetry-api\n",
" Found existing installation: opentelemetry-api 1.37.0\n",
" Uninstalling opentelemetry-api-1.37.0:\n",
" Successfully uninstalled opentelemetry-api-1.37.0\n",
" Attempting uninstall: opentelemetry-semantic-conventions\n",
" Found existing installation: opentelemetry-semantic-conventions 0.58b0\n",
" Uninstalling opentelemetry-semantic-conventions-0.58b0:\n",
" Successfully uninstalled opentelemetry-semantic-conventions-0.58b0\n",
" Attempting uninstall: opentelemetry-sdk\n",
" Found existing installation: opentelemetry-sdk 1.37.0\n",
" Uninstalling opentelemetry-sdk-1.37.0:\n",
" Successfully uninstalled opentelemetry-sdk-1.37.0\n",
"Successfully installed anthropic-0.71.0 backoff-2.2.1 click-8.2.1 deepeval-3.6.2 deepteam-0.2.7 execnet-2.1.1 ollama-0.6.0 opentelemetry-api-1.38.0 opentelemetry-exporter-otlp-proto-common-1.38.0 opentelemetry-exporter-otlp-proto-grpc-1.38.0 opentelemetry-proto-1.38.0 opentelemetry-sdk-1.38.0 opentelemetry-semantic-conventions-0.59b0 portalocker-3.2.0 posthog-6.7.8 pyfiglet-1.0.4 pytest-asyncio-1.2.0 pytest-repeat-0.9.4 pytest-rerunfailures-12.0 pytest-xdist-3.8.0\n"
]
}
],
"source": [
"!pip install deepteam deepeval"
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"\n",
"os.environ[\"OPENAI_API_KEY\"] = \"YOUR_API_KEY\""
],
"metadata": {
"id": "AKBci10zVfTK"
},
"execution_count": 2,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"Want to use other evaluation models? [Click here](https://deepeval.com/integrations/models/openai) to see all supported models and their usage instructions."
],
"metadata": {
"id": "By1mvBjjVv_k"
}
},
{
"cell_type": "code",
"source": [
"from openai import OpenAI\n",
"from deepteam.vulnerabilities import Competition\n",
"from deepteam.attacks.single_turn import PermissionEscalation, PromptInjection\n",
"from deepteam.attacks.multi_turn import LinearJailbreaking, SequentialJailbreak\n",
"from deepteam.test_case import RTTurn\n",
"from deepeval.models import GeminiModel\n",
"\n",
"def model_callback(attack, turn_history=None):\n",
" model = GeminiModel(\n",
" model_name=\"gemini-2.5-flash\",\n",
" api_key=\"YOUR_API_KEY_HERE\",\n",
" temperature=0\n",
" )\n",
" return model.generate(attack)[0]\n",
"\n",
"\n",
"competetion = Competition()\n",
"\n",
"test_cases = competetion.simulate_attacks()\n",
"\n",
"permission_escalation = PermissionEscalation()\n",
"prompt_injection = PromptInjection()\n",
"linear_jailbreaking = LinearJailbreaking()\n",
"sequential_jailbreak = SequentialJailbreak()\n",
"\n",
"safe_model = True\n",
"\n",
"for test_case in test_cases:\n",
" enhanced_attack_1 = permission_escalation.enhance(test_case.input)\n",
" enhanced_attack_2 = prompt_injection.enhance(enhanced_attack_1)\n",
" turns = [RTTurn(role=\"user\", content=enhanced_attack_2)]\n",
" new_turns_1 = linear_jailbreaking._get_turns(\n",
" model_callback, turns, test_case.vulnerability, test_case.vulnerability_type\n",
" )\n",
" new_turns_2 = sequential_jailbreak._get_turns(\n",
" model_callback, new_turns_1, test_case.vulnerability, test_case.vulnerability_type\n",
" )\n",
" test_case.turns = new_turns_2\n",
" metric = competetion._get_metric(test_case.vulnerability_type)\n",
" metric.measure(test_case)\n",
" if metric.score < 1:\n",
" safe_model = False\n",
" print(\"LLM application failed. 😞\")\n",
" print(metric.score)\n",
" print(metric.reason)\n",
" break\n",
"\n",
"if safe_model:\n",
" print(\"LLM application passed! 🥳\")"
],
"metadata": {
"id": "JLGdy0KMVc9v"
},
"execution_count": null,
"outputs": []
}
]
}