DeepTeam is a framework to red team LLMs and AI agents.
Aegis framework added
A-Vamshi435a3cf0f95e
diff --git a/deepteam/dataset/__init__.py b/deepteam/dataset/__init__.pydeleted file mode 100644index 82667637..00000000--- a/deepteam/dataset/__init__.py+++ /dev/null@@ -1,4 +0,0 @@-from .base_dataset import (- BaseRedTeamingDataset,- Attack-)\ No newline at end of filediff --git a/deepteam/dataset/aegis/aegis.py b/deepteam/dataset/aegis/aegis.pydeleted file mode 100644index b3cfacd1..00000000--- a/deepteam/dataset/aegis/aegis.py+++ /dev/null@@ -1,80 +0,0 @@-import asyncio-import random-from typing import Optional, Union-from deepeval.models import DeepEvalBaseLLM-from deepeval.utils import get_or_create_event_loop-from enum import Enum--from deepteam.dataset import (- BaseRedTeamingDataset,- Attack-)-from deepteam.vulnerabilities import BaseVulnerability-from deepteam.test_case import RTTestCase-from deepteam.metrics import DatasetMetric--DATASET="nvidia/Aegis-AI-Content-Safety-Dataset-1.0"--class AegisDataset(BaseVulnerability):- def __init__(- self,- num_attacks: int = 15,- async_mode: bool = True,- verbose_mode: bool = False,- evaluation_model: Optional[Union[str, DeepEvalBaseLLM]] = "gpt-4o",- ):- self.num_attacks = num_attacks- self.async_mode = async_mode- self.verbose_mode = verbose_mode- self.evaluation_model = evaluation_model- self.types = []--- def simulate_attacks(self, purpose=None, attacks_per_vulnerability_type: int = 1,):- from datasets import load_dataset-- dataset = load_dataset(DATASET, split="train")- attacks = [row for row in dataset if row.get("text_type") == "user_message" and row.get("labels_0") != "Safe"]-- if len(attacks) < self.num_attacks:- raise ValueError(f"Requested {self.num_attacks} attacks but only found {len(attacks)} qualifying rows.")-- sampled = random.sample(attacks, k=self.num_attacks)- self.attacks = [Attack(attack=row["text"], category=[row["labels_0"]]) for row in sampled]- test_cases = []- for attack in self.attacks:- vulnerability = ",".join(attack.category)- test_case = RTTestCase(- input=attack.attack,- vulnerability=vulnerability,- )- test_cases.append(test_case)- return test_cases--- async def a_simulate_attacks(self, purpose=None, attacks_per_vulnerability_type: int = 1,):- from datasets import load_dataset-- loop = get_or_create_event_loop()-- def _collect():- dataset = load_dataset(DATASET, split="train")- attacks = [row for row in dataset if row.get("text_type") == "user_message" and row.get("labels_0") != "Safe"]- if len(attacks) < self.num_attacks:- raise ValueError(f"Requested {self.num_attacks} attacks but only found {len(attacks)} qualifying rows.")- sampled = random.sample(attacks, k=self.num_attacks)- return [Attack(attack=row["text"], category=[row["labels_0"]]) for row in sampled]-- self.attacks = await loop.run_in_executor(None, _collect)- return self.attacks--- def _get_metric(self, type=None):- return DatasetMetric(- model=self.evaluation_model,- async_mode=self.async_mode,- verbose_mode=self.verbose_mode,- )-- def get_name(self):- return "Aegis"\ No newline at end of filediff --git a/deepteam/dataset/base_dataset.py b/deepteam/dataset/base_dataset.pydeleted file mode 100644index d2f4420c..00000000--- a/deepteam/dataset/base_dataset.py+++ /dev/null@@ -1,56 +0,0 @@-from pydantic import BaseModel-from typing import Optional, List-from deepteam.metrics import BaseRedTeamingMetric-from abc import ABC--class Attack(BaseModel):- attack: str- category: List[str]---class BaseRedTeamingDataset(ABC):- attacks: Optional[List[Attack]] = None- num_attacks: Optional[int] = None-- def __init__(self):- """- Initialize a RedTeamingDataset.- """- pass-- def get_name(self) -> str:- """- Get the name of Dataset.- :return: A string representing the name of Dataset- """- return self.__class__.__name__-- def assess(self):- pass-- async def a_assess(self):- pass-- def simulate_attacks(self):- pass-- async def a_simulate_attacks(self):- pass-- def _get_metric(self) -> BaseRedTeamingMetric:- """- Get the corresponding metric of the dataset.- :return: The BaseRedTeamingMetric corresponding to this dataset- """- pass-- def get_name(self) -> str:- return self.__class__.__name__-- def __repr__(self):- """- Represent the class by listing the Enum types.- :return: String representation of the Vulnerability class.- """- return f"{self.__class__.__name__} (types={self.types})"-diff --git a/deepteam/frameworks/__init__.py b/deepteam/frameworks/__init__.pyindex 682cf51a..06aef4b2 100644--- a/deepteam/frameworks/__init__.py+++ b/deepteam/frameworks/__init__.py@@ -1,5 +1,8 @@-from .frameworks import OWASPTop10+from .frameworks import OWASPTop10, AISafetyFramework+from .aegis.aegis import Aegis__all__ = [+ "AISafetyFramework","OWASPTop10",+ "Aegis",]diff --git a/deepteam/frameworks/aegis/aegis.py b/deepteam/frameworks/aegis/aegis.pyindex be71c916..84a3efe7 100644--- a/deepteam/frameworks/aegis/aegis.py+++ b/deepteam/frameworks/aegis/aegis.py@@ -1,12 +1,15 @@import asyncioimport random-from typing import Optional, Union+from typing import Optional, Union, Listfrom deepeval.models import DeepEvalBaseLLMfrom deepeval.utils import get_or_create_event_loop+from tqdm import tqdmfrom deepteam.frameworks import AISafetyFramework+from deepteam.attacks.multi_turn.types import CallbackTypefrom deepteam.frameworks.aegis.types import AegisTypefrom deepteam.test_case import RTTestCase+from deepteam.metrics import HarmMetricclass Aegis(AISafetyFramework):_DATASET = "nvidia/Aegis-AI-Content-Safety-Dataset-1.0"@@ -73,5 +76,67 @@ class Aegis(AISafetyFramework):return test_cases+ def evaluate_test_cases(+ self,+ test_cases: List[RTTestCase],+ model_callback: CallbackType+ ):+ evaluated_test_cases = []+ pbar = tqdm(+ total=len(test_cases),+ desc=f"π Evaluating {len(test_cases)} test cases using {self.get_name()} risk categories",+ )+ for test_case in test_cases:+ test_case.actual_output = model_callback(test_case.input)+ metric = self._get_metric(test_case.vulnerability)+ metric.measure(test_case)+ test_case.score = metric.score+ test_case.reason = metric.reason+ evaluated_test_cases.append(test_case)+ pbar.update(1)++ pbar.close()++ return evaluated_test_cases++ async def a_evaluate_test_cases(+ self,+ test_cases: List[RTTestCase],+ model_callback: CallbackType+ ):++ pbar = tqdm(+ total=len(test_cases),+ desc=f"π Evaluating {len(test_cases)} test cases using {self.get_name()} risk categories",+ )++ async def evaluate_test_case(test_case):+ test_case.actual_output = await model_callback(test_case.input)+ metric = self._get_metric(test_case.vulnerability)+ await metric.a_measure(test_case)+ test_case.score = metric.score+ test_case.reason = metric.reason++ pbar.update(1)++ return test_case++ tasks = [evaluate_test_case(tc) for tc in test_cases]++ evaluated_test_cases = await asyncio.gather(*tasks)++ pbar.close()++ return evaluated_test_cases+++ def _get_metric(self, harm_category: str):+ return HarmMetric(+ harm_category=harm_category,+ model=self.evaluation_model,+ async_mode=self.async_mode,+ verbose_mode=self.verbose_mode+ )+def get_name(self):return "Aegis"diff --git a/deepteam/frameworks/frameworks.py b/deepteam/frameworks/frameworks.pyindex 93e9162b..89281bed 100644--- a/deepteam/frameworks/frameworks.py+++ b/deepteam/frameworks/frameworks.py@@ -1,5 +1,7 @@from pydantic import BaseModel-from typing import List+from typing import List, Optional++from deepeval.models import DeepEvalBaseLLMfrom deepteam.frameworks.utils import (_get_owasp_attacks,@@ -8,16 +10,23 @@ from deepteam.frameworks.utils import (from deepteam.vulnerabilities import (BaseVulnerability,)--+from dataclasses import dataclassfrom deepteam.attacks import BaseAttack-class AISafetyFramework(BaseModel):+@dataclass+class AISafetyFramework:name: strdescription: str- vulnerabilities: List[BaseVulnerability]- attacks: List[BaseAttack]+ vulnerabilities: Optional[List[BaseVulnerability]]+ attacks: Optional[List[BaseAttack]]+ _is_dataset: bool = False++ def simulate_attacks(self):+ pass++ async def a_simulate_attacks(self):+ passclass Config:arbitrary_types_allowed = Truediff --git a/deepteam/metrics/dataset_metric/__init__.py b/deepteam/metrics/dataset_metric/__init__.pydeleted file mode 100644index e5d3e904..00000000--- a/deepteam/metrics/dataset_metric/__init__.py+++ /dev/null@@ -1 +0,0 @@-from .schema import ReasonScore\ No newline at end of filediff --git a/deepteam/metrics/dataset_metric/dataset_metric.py b/deepteam/metrics/dataset_metric/dataset_metric.pydeleted file mode 100644index 97b2d723..00000000--- a/deepteam/metrics/dataset_metric/dataset_metric.py+++ /dev/null@@ -1,153 +0,0 @@-from typing import Optional, Tuple, Union--from deepteam.test_case import RTTestCase-from deepeval.utils import get_or_create_event_loop-from deepeval.metrics.utils import (- construct_verbose_logs,- trimAndLoadJson,- initialize_model,-)-from deepeval.models import DeepEvalBaseLLM-from deepeval.metrics.indicator import metric_progress_indicator--from deepteam.metrics import BaseRedTeamingMetric-from deepteam.metrics.harm.template import HarmTemplate-from deepteam.metrics.dataset_metric.schema import *-from deepteam.utils import format_turns---class DatasetMetric(BaseRedTeamingMetric):- def __init__(- self,- model: Optional[Union[str, DeepEvalBaseLLM]] = None,- async_mode: bool = True,- verbose_mode: bool = False,- ):- self.model, self.using_native_model = initialize_model(model)- self.evaluation_model = self.model.get_model_name()- self.async_mode = async_mode- self.verbose_mode = verbose_mode- try:- self.system_prompt = model.get_system_prompt()- except:- self.system_prompt = ""-- def measure(self, test_case: RTTestCase) -> float:-- self.evaluation_cost = 0 if self.using_native_model else None- with metric_progress_indicator(self, _show_indicator=False):- if self.async_mode:- loop = get_or_create_event_loop()- loop.run_until_complete(- self.a_measure(test_case, _show_indicator=False)- )- else:- score, reason = self.evaluate(test_case)- self.reason = reason- self.score = score- self.success = self.score == 1- self.verbose_logs = construct_verbose_logs(- self,- steps=[f"Score: {self.score}\nReason: {self.reason}"],- )- return self.score-- async def a_measure(- self,- test_case: RTTestCase,- _show_indicator: bool = False,- ) -> float:-- self.evaluation_cost = 0 if self.using_native_model else None- with metric_progress_indicator(- self, async_mode=True, _show_indicator=_show_indicator- ):- score, reason = await self._a_evaluate(test_case)- self.reason = reason- self.score = score- self.success = self.score == 1- self.verbose_logs = construct_verbose_logs(- self,- steps=[f"Score: {self.score}\nReason: {self.reason}"],- )- return self.score-- async def _a_evaluate(- self, test_case: RTTestCase- ) -> Tuple[Union[int, float], str]:-- if test_case.turns is not None:- actual_output = format_turns(test_case.turns)- else:- actual_output = (- "AI'S ACTUAL OUTPUT TO EVALUATE: \n"- + test_case.actual_output- + "\n"- )-- prompt = HarmTemplate.generate_evaluation_results(- input=test_case.input,- actual_output=actual_output,- harm_category=test_case.vulnerability- )- if self.using_native_model:- res, cost = await self.model.a_generate(- prompt=prompt, schema=ReasonScore- )- self.evaluation_cost += cost- return res.score, res.reason- else:- try:- res: ReasonScore = await self.model.a_generate(- prompt=prompt, schema=ReasonScore- )- return res.score, res.reason- except TypeError:- res = await self.model.a_generate(prompt)- data = trimAndLoadJson(res, self)- return data["score"], data["reason"]-- def evaluate(self, test_case: RTTestCase) -> Tuple[Union[int, float], str]:-- if test_case.turns is not None:- actual_output = format_turns(test_case.turns)- else:- actual_output = (- "AI'S ACTUAL OUTPUT TO EVALUATE: \n"- + test_case.actual_output- + "\n"- )-- prompt = HarmTemplate.generate_evaluation_results(- input=test_case.input,- actual_output=actual_output,- harm_category=test_case.vulnerability- )- if self.using_native_model:- res, cost = self.model.generate(prompt=prompt, schema=ReasonScore)- self.evaluation_cost += cost- return res.score, res.reason- else:- try:- res: ReasonScore = self.model.generate(- prompt=prompt, schema=ReasonScore- )- return res.score, res.reason- except TypeError:- res = self.model.generate(prompt)- data = trimAndLoadJson(res, self)- return data["score"], data["reason"]-- def is_successful(self) -> bool:- if self.error is not None:- self.success = False- else:- try:- self.score == 1- except:- self.success = False- return self.success-- @property- def __name__(self):- return f"Dataset Metric (Red Teaming)"diff --git a/deepteam/metrics/dataset_metric/schema.py b/deepteam/metrics/dataset_metric/schema.pydeleted file mode 100644index 0ccf4204..00000000--- a/deepteam/metrics/dataset_metric/schema.py+++ /dev/null@@ -1,5 +0,0 @@-from pydantic import BaseModel--class ReasonScore(BaseModel):- reason: str- score: floatdiff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.pyindex ea41e25a..7b3558dd 100644--- a/deepteam/red_teamer/red_teamer.py+++ b/deepteam/red_teamer/red_teamer.py@@ -69,14 +69,14 @@ class RedTeamer:reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,):- if framework:- vulnerabilities = framework.vulnerabilities+ if not framework and not vulnerabilities:+ raise ValueError(+ "You must either provide a 'framework' or 'vulnerabilities'"+ )++ if framework and framework._is_dataset is False:attacks = framework.attacks- else:- if not vulnerabilities:- raise ValueError(- "You must either provide a 'framework' or 'vulnerabilities'"- )+ vulnerabilities = framework.vulnerabilitiesif self.async_mode:validate_model_callback_signature(@@ -89,6 +89,7 @@ class RedTeamer:model_callback=model_callback,attacks_per_vulnerability_type=attacks_per_vulnerability_type,vulnerabilities=vulnerabilities,+ framework=framework,attacks=attacks,simulator_model=simulator_model,evaluation_model=evaluation_model,@@ -107,7 +108,7 @@ class RedTeamer:if simulator_model is not None:self.simulator_model = simulator_modelwith capture_red_teamer_run(- vulnerabilities=[v.get_name() for v in vulnerabilities],+ vulnerabilities=[v.get_name() for v in vulnerabilities] if vulnerabilities else [f"Framework: {framework.get_name()}"],attacks=[a.get_name() for a in attacks] if attacks else [],):# Generate attacks@@ -123,6 +124,7 @@ class RedTeamer:self.attack_simulator.simulate(attacks_per_vulnerability_type=attacks_per_vulnerability_type,vulnerabilities=vulnerabilities,+ framework=framework,attacks=attacks,ignore_errors=ignore_errors,simulator_model=self.simulator_model,@@ -154,36 +156,42 @@ class RedTeamer:simulated_test_case.risk_category.value)- total_attacks = sum(- len(attacks)- for attacks in vulnerability_type_to_attacks_map.values()- )- num_vulnerability_types = sum(- len(v.get_types()) for v in vulnerabilities- )- pbar = tqdm(- total=total_attacks,- desc=f"π Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",- )+ if framework and framework._is_dataset:+ red_teaming_test_cases = framework.evaluate_test_cases(+ test_cases=simulated_test_cases,+ model_callback=model_callback,+ )+ else:+ total_attacks = sum(+ len(test_cases)+ for test_cases in vulnerability_type_to_attacks_map.values()+ )+ num_vulnerability_types = sum(+ len(v.get_types()) for v in vulnerabilities+ )+ pbar = tqdm(+ total=total_attacks,+ desc=f"π Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",+ )- red_teaming_test_cases: List[RTTestCase] = []+ red_teaming_test_cases: List[RTTestCase] = []- for (- vulnerability_type,- attacks,- ) in vulnerability_type_to_attacks_map.items():- test_cases = self._evaluate_vulnerability_type(- model_callback,- vulnerabilities,+ for (vulnerability_type,- attacks,- ignore_errors=ignore_errors,- )- red_teaming_test_cases.extend(test_cases)+ test_cases,+ ) in vulnerability_type_to_attacks_map.items():+ rt_test_cases = self._evaluate_vulnerability_type(+ model_callback,+ vulnerabilities,+ vulnerability_type,+ test_cases,+ ignore_errors=ignore_errors,+ )+ red_teaming_test_cases.extend(rt_test_cases)- pbar.update(len(attacks))+ pbar.update(len(test_cases))- pbar.close()+ pbar.close()self.risk_assessment = RiskAssessment(overview=construct_risk_assessment_overview(@@ -209,14 +217,14 @@ class RedTeamer:reuse_simulated_test_cases: bool = False,metadata: Optional[dict] = None,):- if framework:- vulnerabilities = framework.vulnerabilities+ if not framework and not vulnerabilities:+ raise ValueError(+ "You must either provide a 'framework' or 'vulnerabilities'"+ )++ if framework and framework._is_dataset is False:attacks = framework.attacks- else:- if not vulnerabilities:- raise ValueError(- "You must either provide a 'framework' or 'vulnerabilities'"- )+ vulnerabilities = framework.vulnerabilitiesif evaluation_model is not None:self.evaluation_model = evaluation_model@@ -224,7 +232,7 @@ class RedTeamer:self.simulator_model = simulator_modelwith capture_red_teamer_run(- vulnerabilities=[v.get_name() for v in vulnerabilities],+ vulnerabilities=[v.get_name() for v in vulnerabilities] if vulnerabilities else [f"Framework: {framework.get_name()}"],attacks=[a.get_name() for a in attacks] if attacks else [],):# Generate attacks@@ -240,6 +248,7 @@ class RedTeamer:await self.attack_simulator.a_simulate(attacks_per_vulnerability_type=attacks_per_vulnerability_type,vulnerabilities=vulnerabilities,+ framework=framework,attacks=attacks,simulator_model=self.simulator_model,ignore_errors=ignore_errors,@@ -271,44 +280,50 @@ class RedTeamer:simulated_test_case.risk_category.value)- semaphore = asyncio.Semaphore(self.max_concurrent)- total_attacks = sum(- len(attacks)- for attacks in vulnerability_type_to_attacks_map.values()- )- num_vulnerability_types = sum(- len(v.get_types()) for v in vulnerabilities- )- pbar = tqdm(- total=total_attacks,- desc=f"π Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",- )-- red_teaming_test_cases: List[RTTestCase] = []+ if framework and framework._is_dataset:+ red_teaming_test_cases = await framework.a_evaluate_test_cases(+ test_cases=simulated_test_cases,+ model_callback=model_callback,+ )+ else:+ semaphore = asyncio.Semaphore(self.max_concurrent)+ total_attacks = sum(+ len(attacks)+ for attacks in vulnerability_type_to_attacks_map.values()+ )+ num_vulnerability_types = sum(+ len(v.get_types()) for v in vulnerabilities+ )+ pbar = tqdm(+ total=total_attacks,+ desc=f"π Evaluating {num_vulnerability_types} vulnerability types across {len(vulnerabilities)} vulnerability(s)",+ )- async def throttled_evaluate_vulnerability_type(- vulnerability_type, attacks- ):- async with semaphore:- test_cases = await self._a_evaluate_vulnerability_type(- model_callback,- vulnerabilities,- vulnerability_type,- attacks,- ignore_errors=ignore_errors,- )- red_teaming_test_cases.extend(test_cases)- pbar.update(len(attacks))+ red_teaming_test_cases: List[RTTestCase] = []- # Create a list of tasks for evaluating each vulnerability, with throttling- tasks = [- throttled_evaluate_vulnerability_type(+ async def throttled_evaluate_vulnerability_type(vulnerability_type, attacks- )- for vulnerability_type, attacks in vulnerability_type_to_attacks_map.items()- ]- await asyncio.gather(*tasks)- pbar.close()+ ):+ async with semaphore:+ test_cases = await self._a_evaluate_vulnerability_type(+ model_callback,+ vulnerabilities,+ vulnerability_type,+ attacks,+ ignore_errors=ignore_errors,+ )+ red_teaming_test_cases.extend(test_cases)+ pbar.update(len(attacks))++ # Create a list of tasks for evaluating each vulnerability, with throttling+ tasks = [+ throttled_evaluate_vulnerability_type(+ vulnerability_type, attacks+ )+ for vulnerability_type, attacks in vulnerability_type_to_attacks_map.items()+ ]+ await asyncio.gather(*tasks)+ pbar.close()self.risk_assessment = RiskAssessment(overview=construct_risk_assessment_overview(