Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

Aegis framework added

A-Vamshi435a3cf0f95e
diff --git a/deepteam/dataset/__init__.py b/deepteam/dataset/__init__.py
deleted file mode 100644
index 82667637..00000000
--- a/deepteam/dataset/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from .base_dataset import (
- BaseRedTeamingDataset,
- Attack
-)
\ No newline at end of file
diff --git a/deepteam/dataset/aegis/aegis.py b/deepteam/dataset/aegis/aegis.py
deleted file mode 100644
index b3cfacd1..00000000
--- a/deepteam/dataset/aegis/aegis.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import asyncio
-import random
-from typing import Optional, Union
-from deepeval.models import DeepEvalBaseLLM
-from deepeval.utils import get_or_create_event_loop
-from enum import Enum
-
-from deepteam.dataset import (
- BaseRedTeamingDataset,
- Attack
-)
-from deepteam.vulnerabilities import BaseVulnerability
-from deepteam.test_case import RTTestCase
-from deepteam.metrics import DatasetMetric
-
-DATASET="nvidia/Aegis-AI-Content-Safety-Dataset-1.0"
-
-class AegisDataset(BaseVulnerability):
- def __init__(
- self,
- num_attacks: int = 15,
- async_mode: bool = True,
- verbose_mode: bool = False,
- evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",
- ):
- self.num_attacks = num_attacks
- self.async_mode = async_mode
- self.verbose_mode = verbose_mode
- self.evaluation_model = evaluation_model
- self.types = []
-
-
- def simulate_attacks(self, purpose=None, attacks_per_vulnerability_type: int = 1,):
- from datasets import load_dataset
-
- dataset = load_dataset(DATASET, split="train")
- attacks = [row for row in dataset if row.get("text_type") == "user_message" and row.get("labels_0") != "Safe"]
-
- if len(attacks) < self.num_attacks:
- raise ValueError(f"Requested {self.num_attacks} attacks but only found {len(attacks)} qualifying rows.")
-
- sampled = random.sample(attacks, k=self.num_attacks)
- self.attacks = [Attack(attack=row["text"], category=[row["labels_0"]]) for row in sampled]
- test_cases = []
- for attack in self.attacks:
- vulnerability = ",".join(attack.category)
- test_case = RTTestCase(
- input=attack.attack,
- vulnerability=vulnerability,
- )
- test_cases.append(test_case)
- return test_cases
-
-
- async def a_simulate_attacks(self, purpose=None, attacks_per_vulnerability_type: int = 1,):
- from datasets import load_dataset
-
- loop = get_or_create_event_loop()
-
- def _collect():
- dataset = load_dataset(DATASET, split="train")
- attacks = [row for row in dataset if row.get("text_type") == "user_message" and row.get("labels_0") != "Safe"]
- if len(attacks) < self.num_attacks:
- raise ValueError(f"Requested {self.num_attacks} attacks but only found {len(attacks)} qualifying rows.")
- sampled = random.sample(attacks, k=self.num_attacks)
- return [Attack(attack=row["text"], category=[row["labels_0"]]) for row in sampled]
-
- self.attacks = await loop.run_in_executor(None, _collect)
- return self.attacks
-
-
- def _get_metric(self, type=None):
- return DatasetMetric(
- model=self.evaluation_model,
- async_mode=self.async_mode,
- verbose_mode=self.verbose_mode,
- )
-
- def get_name(self):
- return "Aegis"
\ No newline at end of file
diff --git a/deepteam/dataset/base_dataset.py b/deepteam/dataset/base_dataset.py
deleted file mode 100644
index d2f4420c..00000000
--- a/deepteam/dataset/base_dataset.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from pydantic import BaseModel
-from typing import Optional, List
-from deepteam.metrics import BaseRedTeamingMetric
-from abc import ABC
-
-class Attack(BaseModel):
- attack: str
- category: List[str]
-
-
-class BaseRedTeamingDataset(ABC):
- attacks: Optional[List[Attack]] = None
- num_attacks: Optional[int] = None
-
- def __init__(self):
- """
- Initialize a RedTeamingDataset.
- """
- pass
-
- def get_name(self) -> str:
- """
- Get the name of Dataset.
- :return: A string representing the name of Dataset
- """
- return self.__class__.__name__
-
- def assess(self):
- pass
-
- async def a_assess(self):
- pass
-
- def simulate_attacks(self):
- pass
-
- async def a_simulate_attacks(self):
- pass
-
- def _get_metric(self) -> BaseRedTeamingMetric:
- """
- Get the corresponding metric of the dataset.
- :return: The BaseRedTeamingMetric corresponding to this dataset
- """
- pass
-
- def get_name(self) -> str:
- return self.__class__.__name__
-
- def __repr__(self):
- """
- Represent the class by listing the Enum types.
- :return: String representation of the Vulnerability class.
- """
- return f"{self.__class__.__name__} (types={self.types})"
-
diff --git a/deepteam/frameworks/__init__.py b/deepteam/frameworks/__init__.py
index 682cf51a..06aef4b2 100644
--- a/deepteam/frameworks/__init__.py
+++ b/deepteam/frameworks/__init__.py
@@ -1,5 +1,8 @@
-from .frameworks import OWASPTop10
+from .frameworks import OWASPTop10, AISafetyFramework
+from .aegis.aegis import Aegis
__all__ = [
+ "AISafetyFramework",
"OWASPTop10",
+ "Aegis",
]
diff --git a/deepteam/frameworks/aegis/aegis.py b/deepteam/frameworks/aegis/aegis.py
index be71c916..84a3efe7 100644
--- a/deepteam/frameworks/aegis/aegis.py
+++ b/deepteam/frameworks/aegis/aegis.py
@@ -1,12 +1,15 @@
import asyncio
import random
-from typing import Optional, Union
+from typing import Optional, Union, List
from deepeval.models import DeepEvalBaseLLM
from deepeval.utils import get_or_create_event_loop
+from tqdm import tqdm
from deepteam.frameworks import AISafetyFramework
+from deepteam.attacks.multi_turn.types import CallbackType
from deepteam.frameworks.aegis.types import AegisType
from deepteam.test_case import RTTestCase
+from deepteam.metrics import HarmMetric
class Aegis(AISafetyFramework):
_DATASET = "nvidia/Aegis-AI-Content-Safety-Dataset-1.0"
@@ -73,5 +76,67 @@ class Aegis(AISafetyFramework):
return test_cases
+ def evaluate_test_cases(
+ self,
+ test_cases: List[RTTestCase],
+ model_callback: CallbackType
+ ):
+ evaluated_test_cases = []
+ pbar = tqdm(
+ total=len(test_cases),
+ desc=f"πŸ“ Evaluating {len(test_cases)} test cases using {self.get_name()} risk categories",
+ )
+ for test_case in test_cases:
+ test_case.actual_output = model_callback(test_case.input)
+ metric = self._get_metric(test_case.vulnerability)
+ metric.measure(test_case)
+ test_case.score = metric.score
+ test_case.reason = metric.reason
+ evaluated_test_cases.append(test_case)
+ pbar.update(1)
+
+ pbar.close()
+
+ return evaluated_test_cases
+
+ async def a_evaluate_test_cases(
+ self,
+ test_cases: List[RTTestCase],
+ model_callback: CallbackType
+ ):
+
+ pbar = tqdm(
+ total=len(test_cases),
+ desc=f"πŸ“ Evaluating {len(test_cases)} test cases using {self.get_name()} risk categories",
+ )
+
+ async def evaluate_test_case(test_case):
+ test_case.actual_output = await model_callback(test_case.input)
+ metric = self._get_metric(test_case.vulnerability)
+ await metric.a_measure(test_case)
+ test_case.score = metric.score
+ test_case.reason = metric.reason
+
+ pbar.update(1)
+
+ return test_case
+
+ tasks = [evaluate_test_case(tc) for tc in test_cases]
+
+ evaluated_test_cases = await asyncio.gather(*tasks)
+
+ pbar.close()
+
+ return evaluated_test_cases
+
+
+ def _get_metric(self, harm_category: str):
+ return HarmMetric(
+ harm_category=harm_category,
+ model=self.evaluation_model,
+ async_mode=self.async_mode,
+ verbose_mode=self.verbose_mode
+ )
+
def get_name(self):
return "Aegis"
diff --git a/deepteam/frameworks/frameworks.py b/deepteam/frameworks/frameworks.py
index 93e9162b..89281bed 100644
--- a/deepteam/frameworks/frameworks.py
+++ b/deepteam/frameworks/frameworks.py
@@ -1,5 +1,7 @@
from pydantic import BaseModel
-from typing import List
+from typing import List, Optional
+
+from deepeval.models import DeepEvalBaseLLM
from deepteam.frameworks.utils import (
_get_owasp_attacks,
@@ -8,16 +10,23 @@ from deepteam.frameworks.utils import (
from deepteam.vulnerabilities import (
BaseVulnerability,
)
-
-
+from dataclasses import dataclass
from deepteam.attacks import BaseAttack
-class AISafetyFramework(BaseModel):
+@dataclass
+class AISafetyFramework:
name: str
description: str
- vulnerabilities: List[BaseVulnerability]
- attacks: List[BaseAttack]
+ vulnerabilities: Optional[List[BaseVulnerability]]
+ attacks: Optional[List[BaseAttack]]
+ _is_dataset: bool = False
+
+ def simulate_attacks(self):
+ pass
+
+ async def a_simulate_attacks(self):
+ pass
class Config:
arbitrary_types_allowed = True
diff --git a/deepteam/metrics/dataset_metric/__init__.py b/deepteam/metrics/dataset_metric/__init__.py
deleted file mode 100644
index e5d3e904..00000000
--- a/deepteam/metrics/dataset_metric/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .schema import ReasonScore
\ No newline at end of file
diff --git a/deepteam/metrics/dataset_metric/dataset_metric.py b/deepteam/metrics/dataset_metric/dataset_metric.py
deleted file mode 100644
index 97b2d723..00000000
--- a/deepteam/metrics/dataset_metric/dataset_metric.py
+++ /dev/null
@@ -1,153 +0,0 @@
-from typing import Optional, Tuple, Union
-
-from deepteam.test_case import RTTestCase
-from deepeval.utils import get_or_create_event_loop
-from deepeval.metrics.utils import (
- construct_verbose_logs,
- trimAndLoadJson,
- initialize_model,
-)
-from deepeval.models import DeepEvalBaseLLM
-from deepeval.metrics.indicator import metric_progress_indicator
-
-from deepteam.metrics import BaseRedTeamingMetric
-from deepteam.metrics.harm.template import HarmTemplate
-from deepteam.metrics.dataset_metric.schema import *
-from deepteam.utils import format_turns
-
-
-class DatasetMetric(BaseRedTeamingMetric):
- def __init__(
- self,
- model: Optional[Union[str, DeepEvalBaseLLM]] = None,
- async_mode: bool = True,
- verbose_mode: bool = False,
- ):
- self.model, self.using_native_model = initialize_model(model)
- self.evaluation_model = self.model.get_model_name()
- self.async_mode = async_mode
- self.verbose_mode = verbose_mode
- try:
- self.system_prompt = model.get_system_prompt()
- except:
- self.system_prompt = ""
-
- def measure(self, test_case: RTTestCase) -> float:
-
- self.evaluation_cost = 0 if self.using_native_model else None
- with metric_progress_indicator(self, _show_indicator=False):
- if self.async_mode:
- loop = get_or_create_event_loop()
- loop.run_until_complete(
- self.a_measure(test_case, _show_indicator=False)
- )
- else:
- score, reason = self.evaluate(test_case)
- self.reason = reason
- self.score = score
- self.success = self.score == 1
- self.verbose_logs = construct_verbose_logs(
- self,
- steps=[f"Score: {self.score}\nReason: {self.reason}"],
- )
- return self.score
-
- async def a_measure(
- self,
- test_case: RTTestCase,
- _show_indicator: bool = False,
- ) -> float:
-
- self.evaluation_cost = 0 if self.using_native_model else None
- with metric_progress_indicator(
- self, async_mode=True, _show_indicator=_show_indicator
- ):
- score, reason = await self._a_evaluate(test_case)
- self.reason = reason
- self.score = score
- self.success = self.score == 1
- self.verbose_logs = construct_verbose_logs(
- self,
- steps=[f"Score: {self.score}\nReason: {self.reason}"],
- )
- return self.score
-
- async def _a_evaluate(
- self, test_case: RTTestCase
- ) -> Tuple[Union[int, float], str]:
-
- if test_case.turns is not None:
- actual_output = format_turns(test_case.turns)
- else:
- actual_output = (
- "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
- + test_case.actual_output
- + "\n"
- )
-
- prompt = HarmTemplate.generate_evaluation_results(
- input=test_case.input,
- actual_output=actual_output,
- harm_category=test_case.vulnerability
- )
- if self.using_native_model:
- res, cost = await self.model.a_generate(
- prompt=prompt, schema=ReasonScore
- )
- self.evaluation_cost += cost
- return res.score, res.reason
- else:
- try:
- res: ReasonScore = await self.model.a_generate(
- prompt=prompt, schema=ReasonScore
- )
- return res.score, res.reason
- except TypeError:
- res = await self.model.a_generate(prompt)
- data = trimAndLoadJson(res, self)
- return data["score"], data["reason"]
-
- def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:
-
- if test_case.turns is not None:
- actual_output = format_turns(test_case.turns)
- else:
- actual_output = (
- "AI'S ACTUAL OUTPUT TO EVALUATE: \n"
- + test_case.actual_output
- + "\n"
- )
-
- prompt = HarmTemplate.generate_evaluation_results(
- input=test_case.input,
- actual_output=actual_output,
- harm_category=test_case.vulnerability
- )
- if self.using_native_model:
- res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)
- self.evaluation_cost += cost
- return res.score, res.reason
- else:
- try:
- res: ReasonScore = self.model.generate(
- prompt=prompt, schema=ReasonScore
- )
- return res.score, res.reason
- except TypeError:
- res = self.model.generate(prompt)
- data = trimAndLoadJson(res, self)
- return data["score"], data["reason"]
-
- def is_successful(self) -> bool:
- if self.error is not None:
- self.success = False
- else:
- try:
- self.score == 1
- except:
- self.success = False
- return self.success
-
- @property
- def __name__(self):
- return f"Dataset Metric (Red Teaming)"
diff --git a/deepteam/metrics/dataset_metric/schema.py b/deepteam/metrics/dataset_metric/schema.py
deleted file mode 100644
index 0ccf4204..00000000
--- a/deepteam/metrics/dataset_metric/schema.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from pydantic import BaseModel
-
-class ReasonScore(BaseModel):
- reason: str
- score: float
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index ea41e25a..7b3558dd 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -69,14 +69,14 @@ class RedTeamer:
reuse_simulated_test_cases: bool = False,
metadata: Optional[dict] = None,
):
- if framework:
- vulnerabilities = framework.vulnerabilities
+ if not framework and not vulnerabilities:
+ raise ValueError(
+ "You must either provide a 'framework' or 'vulnerabilities'"
+ )
+
+ if framework and framework._is_dataset is False:
attacks = framework.attacks
- else:
- if not vulnerabilities:
- raise ValueError(
- "You must either provide a 'framework' or 'vulnerabilities'"
- )
+ vulnerabilities = framework.vulnerabilities
if self.async_mode:
validate_model_callback_signature(
@@ -89,6 +89,7 @@ class RedTeamer:
model_callback=model_callback,
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
vulnerabilities=vulnerabilities,
+ framework=framework,
attacks=attacks,
simulator_model=simulator_model,
evaluation_model=evaluation_model,
@@ -107,7 +108,7 @@ class RedTeamer:
if simulator_model is not None:
self.simulator_model = simulator_model
with capture_red_teamer_run(
- vulnerabilities=[v.get_name() for v in vulnerabilities],
+ vulnerabilities=[v.get_name() for v in vulnerabilities] if vulnerabilities else [f"Framework: {framework.get_name()}"],
attacks=[a.get_name() for a in attacks] if attacks else [],
):
# Generate attacks
@@ -123,6 +124,7 @@ class RedTeamer:
self.attack_simulator.simulate(
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
vulnerabilities=vulnerabilities,
+ framework=framework,
attacks=attacks,
ignore_errors=ignore_errors,
simulator_model=self.simulator_model,
@@ -154,36 +156,42 @@ class RedTeamer:
simulated_test_case.risk_category.value
)
- total_attacks = sum(
- len(attacks)
- for attacks in vulnerability_type_to_attacks_map.values()
- )
- num_vulnerability_types = sum(
- len(v.get_types()) for v in vulnerabilities
- )
- pbar = tqdm(
- total=total_attacks,
- desc=f"πŸ“ Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",
- )
+ if framework and framework._is_dataset:
+ red_teaming_test_cases = framework.evaluate_test_cases(
+ test_cases=simulated_test_cases,
+ model_callback=model_callback,
+ )
+ else:
+ total_attacks = sum(
+ len(test_cases)
+ for test_cases in vulnerability_type_to_attacks_map.values()
+ )
+ num_vulnerability_types = sum(
+ len(v.get_types()) for v in vulnerabilities
+ )
+ pbar = tqdm(
+ total=total_attacks,
+ desc=f"πŸ“ Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",
+ )
- red_teaming_test_cases: List[RTTestCase] = []
+ red_teaming_test_cases: List[RTTestCase] = []
- for (
- vulnerability_type,
- attacks,
- ) in vulnerability_type_to_attacks_map.items():
- test_cases = self._evaluate_vulnerability_type(
- model_callback,
- vulnerabilities,
+ for (
vulnerability_type,
- attacks,
- ignore_errors=ignore_errors,
- )
- red_teaming_test_cases.extend(test_cases)
+ test_cases,
+ ) in vulnerability_type_to_attacks_map.items():
+ rt_test_cases = self._evaluate_vulnerability_type(
+ model_callback,
+ vulnerabilities,
+ vulnerability_type,
+ test_cases,
+ ignore_errors=ignore_errors,
+ )
+ red_teaming_test_cases.extend(rt_test_cases)
- pbar.update(len(attacks))
+ pbar.update(len(test_cases))
- pbar.close()
+ pbar.close()
self.risk_assessment = RiskAssessment(
overview=construct_risk_assessment_overview(
@@ -209,14 +217,14 @@ class RedTeamer:
reuse_simulated_test_cases: bool = False,
metadata: Optional[dict] = None,
):
- if framework:
- vulnerabilities = framework.vulnerabilities
+ if not framework and not vulnerabilities:
+ raise ValueError(
+ "You must either provide a 'framework' or 'vulnerabilities'"
+ )
+
+ if framework and framework._is_dataset is False:
attacks = framework.attacks
- else:
- if not vulnerabilities:
- raise ValueError(
- "You must either provide a 'framework' or 'vulnerabilities'"
- )
+ vulnerabilities = framework.vulnerabilities
if evaluation_model is not None:
self.evaluation_model = evaluation_model
@@ -224,7 +232,7 @@ class RedTeamer:
self.simulator_model = simulator_model
with capture_red_teamer_run(
- vulnerabilities=[v.get_name() for v in vulnerabilities],
+ vulnerabilities=[v.get_name() for v in vulnerabilities] if vulnerabilities else [f"Framework: {framework.get_name()}"],
attacks=[a.get_name() for a in attacks] if attacks else [],
):
# Generate attacks
@@ -240,6 +248,7 @@ class RedTeamer:
await self.attack_simulator.a_simulate(
attacks_per_vulnerability_type=attacks_per_vulnerability_type,
vulnerabilities=vulnerabilities,
+ framework=framework,
attacks=attacks,
simulator_model=self.simulator_model,
ignore_errors=ignore_errors,
@@ -271,44 +280,50 @@ class RedTeamer:
simulated_test_case.risk_category.value
)
- semaphore = asyncio.Semaphore(self.max_concurrent)
- total_attacks = sum(
- len(attacks)
- for attacks in vulnerability_type_to_attacks_map.values()
- )
- num_vulnerability_types = sum(
- len(v.get_types()) for v in vulnerabilities
- )
- pbar = tqdm(
- total=total_attacks,
- desc=f"πŸ“ Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",
- )
-
- red_teaming_test_cases: List[RTTestCase] = []
+ if framework and framework._is_dataset:
+ red_teaming_test_cases = await framework.a_evaluate_test_cases(
+ test_cases=simulated_test_cases,
+ model_callback=model_callback,
+ )
+ else:
+ semaphore = asyncio.Semaphore(self.max_concurrent)
+ total_attacks = sum(
+ len(attacks)
+ for attacks in vulnerability_type_to_attacks_map.values()
+ )
+ num_vulnerability_types = sum(
+ len(v.get_types()) for v in vulnerabilities
+ )
+ pbar = tqdm(
+ total=total_attacks,
+ desc=f"πŸ“ Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",
+ )
- async def throttled_evaluate_vulnerability_type(
- vulnerability_type, attacks
- ):
- async with semaphore:
- test_cases = await self._a_evaluate_vulnerability_type(
- model_callback,
- vulnerabilities,
- vulnerability_type,
- attacks,
- ignore_errors=ignore_errors,
- )
- red_teaming_test_cases.extend(test_cases)
- pbar.update(len(attacks))
+ red_teaming_test_cases: List[RTTestCase] = []
- # Create a list of tasks for evaluating each vulnerability, with throttling
- tasks = [
- throttled_evaluate_vulnerability_type(
+ async def throttled_evaluate_vulnerability_type(
vulnerability_type, attacks
- )
- for vulnerability_type, attacks in vulnerability_type_to_attacks_map.items()
- ]
- await asyncio.gather(*tasks)
- pbar.close()
+ ):
+ async with semaphore:
+ test_cases = await self._a_evaluate_vulnerability_type(
+ model_callback,
+ vulnerabilities,
+ vulnerability_type,
+ attacks,
+ ignore_errors=ignore_errors,
+ )
+ red_teaming_test_cases.extend(test_cases)
+ pbar.update(len(attacks))
+
+ # Create a list of tasks for evaluating each vulnerability, with throttling
+ tasks = [
+ throttled_evaluate_vulnerability_type(
+ vulnerability_type, attacks
+ )
+ for vulnerability_type, attacks in vulnerability_type_to_attacks_map.items()
+ ]
+ await asyncio.gather(*tasks)
+ pbar.close()
self.risk_assessment = RiskAssessment(
overview=construct_risk_assessment_overview(