Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

add pull and upload method to custom vulnerability

A-Vamshicbccdcf624ef
diff --git a/deepteam/confident/api.py b/deepteam/confident/api.py
index 7710a0d0..f38b1b0b 100644
--- a/deepteam/confident/api.py
+++ b/deepteam/confident/api.py
@@ -25,6 +25,8 @@ from deepeval.confident.api import (
class Endpoints(Enum):
RISK_ASSESSMENT_ENDPOINT = "/v1/risk-assessments"
RT_FRAMEWORK_ENDPOINT = "/v1/rt-frameworks/:frameworkId"
+ VULNERABILITIES_ENDPOINT = "/v1/vulnerabilities"
+ VULNERABILITY_ENDPOINT = "/v1/vulnerabilities/:vulnerabilityId"
class Api:
diff --git a/deepteam/vulnerabilities/custom/api.py b/deepteam/vulnerabilities/custom/api.py
new file mode 100644
index 00000000..17d553cd
--- /dev/null
+++ b/deepteam/vulnerabilities/custom/api.py
@@ -0,0 +1,43 @@
+from typing import List, Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class APIEvaluationExample(BaseModel):
+ input: str
+ actual_output: str = Field(alias="actualOutput")
+ score: Literal[0, 1]
+ reason: str
+
+
+class APIVulnerabilityType(BaseModel):
+ id: str
+ name: str
+
+
+class CustomVulnerabilityHttpResponse(BaseModel):
+ id: str
+ name: str
+ description: Optional[str] = None
+ built_in: bool = Field(alias="builtIn")
+ criteria: Optional[str] = None
+ evaluation_guidelines: List[str] = Field(
+ default_factory=list, alias="evaluationGuidelines"
+ )
+ evaluation_examples: List[APIEvaluationExample] = Field(
+ default_factory=list, alias="evaluationExamples"
+ )
+ vulnerability_types: List[APIVulnerabilityType] = Field(
+ alias="vulnerabilityTypes"
+ )
+
+
+class CustomVulnerabilityUploadRequest(BaseModel):
+ name: str
+ criteria: str
+ vulnerability_types: List[str] = Field(alias="vulnerabilityTypes")
+ evaluation_guidelines: Optional[List[str]] = Field(
+ default=None, alias="evaluationGuidelines"
+ )
+ evaluation_examples: Optional[List[APIEvaluationExample]] = Field(
+ default=None, alias="evaluationExamples"
+ )
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index 1af3e1f7..3e681abe 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -7,6 +7,9 @@ from deepeval.metrics.utils import initialize_model, trimAndLoadJson
from deepeval.utils import get_or_create_event_loop
from deepteam.utils import validate_model_callback_signature
+from deepteam.confident.api import Api, Endpoints
+from deepeval.confident.api import HttpMethods
+from rich.console import Console
from deepteam.vulnerabilities import BaseVulnerability
from deepteam.metrics import BaseRedTeamingMetric, HarmMetric
@@ -17,11 +20,26 @@ from deepteam.attacks.attack_simulator.schema import SyntheticDataList
from deepteam.risks import getRiskCategory
from deepteam.test_case import RTTestCase
from .template import CustomVulnerabilityTemplate
+from .api import (
+ APIEvaluationExample,
+ CustomVulnerabilityHttpResponse,
+ CustomVulnerabilityUploadRequest,
+)
from deepeval.tracing.types import Trace
from deepteam.trace_scanner.schema import BatchFinding
from deepteam.trace_scanner import TraceScanner
+def _build_types(name: str, types: Optional[List[str]]) -> List[Enum]:
+ if types:
+ members = Enum("CustomVulnerabilityType", {t.upper(): t for t in types})
+ else:
+ members = Enum(
+ "CustomVulnerabilityType", {name.upper().replace(" ", "_"): name}
+ )
+ return list(members)
+
+
class CustomVulnerability(BaseVulnerability):
"""
Custom vulnerability class that allows users to define their own vulnerability types.
@@ -30,7 +48,7 @@ class CustomVulnerability(BaseVulnerability):
def __init__(
self,
name: str,
- criteria: str,
+ criteria: Optional[str] = None,
types: Optional[List[str]] = None,
custom_prompt: Optional[str] = None,
async_mode: bool = True,
@@ -45,20 +63,10 @@ class CustomVulnerability(BaseVulnerability):
):
self.name = name
- if types:
- self.types = Enum(
- f"CustomVulnerabilityType", {t.upper(): t for t in types}
- )
- else:
- # Default to a single type derived from the vulnerability name
- # so iteration over self.types always works
- self.types = Enum(
- f"CustomVulnerabilityType",
- {name.upper().replace(" ", "_"): name},
- )
+ self.types = _build_types(name, types)
self.custom_prompt = custom_prompt
- self.criteria = criteria.strip()
+ self.criteria = criteria.strip() if criteria else None
self.simulator_model = simulator_model
self.evaluation_model = evaluation_model
self.evaluation_examples = evaluation_examples
@@ -75,6 +83,14 @@ class CustomVulnerability(BaseVulnerability):
def get_custom_prompt(self) -> Optional[str]:
return self.custom_prompt
+ def _ensure_criteria(self) -> str:
+ if not self.criteria:
+ raise ValueError(
+ f"Custom vulnerability '{self.name}' has no criteria. Pass criteria= when "
+ "constructing it, or call pull() to load it from Confident AI."
+ )
+ return self.criteria
+
def assess(
self,
model_callback: CallbackType,
@@ -194,6 +210,7 @@ class CustomVulnerability(BaseVulnerability):
purpose: str = None,
attacks_per_vulnerability_type: int = 1,
) -> List[RTTestCase]:
+ self._ensure_criteria()
self.purpose = purpose
@@ -259,6 +276,7 @@ class CustomVulnerability(BaseVulnerability):
purpose: str = None,
attacks_per_vulnerability_type: int = 1,
) -> List[RTTestCase]:
+ self._ensure_criteria()
self.purpose = purpose
@@ -329,7 +347,7 @@ class CustomVulnerability(BaseVulnerability):
return (
CustomVulnerabilityTemplate.custom_vulnerability_template_wrapper(
name=self.name,
- criteria=self.criteria,
+ criteria=self._ensure_criteria(),
type_values=self.get_values(),
)
)
@@ -390,7 +408,7 @@ class CustomVulnerability(BaseVulnerability):
def _get_metric(self, type: Enum) -> BaseRedTeamingMetric:
if self.metric is None:
self.metric = HarmMetric(
- harm_category=self.criteria,
+ harm_category=self._ensure_criteria(),
model=self.evaluation_model,
async_mode=self.async_mode,
verbose_mode=self.verbose_mode,
@@ -409,5 +427,89 @@ class CustomVulnerability(BaseVulnerability):
self.vulnerable = False
return self.vulnerable
- def get_criteria(self) -> str:
+ def get_criteria(self) -> Optional[str]:
return self.criteria
+
+ def upload(self) -> str:
+ request = CustomVulnerabilityUploadRequest(
+ name=self.name,
+ criteria=self._ensure_criteria(),
+ vulnerabilityTypes=self.get_values(),
+ evaluationGuidelines=self.evaluation_guidelines,
+ evaluationExamples=(
+ [
+ APIEvaluationExample(
+ input=example.input,
+ actualOutput=example.actual_output,
+ score=example.score,
+ reason=example.reason,
+ )
+ for example in self.evaluation_examples
+ ]
+ if self.evaluation_examples is not None
+ else None
+ ),
+ )
+
+ try:
+ body = request.model_dump(by_alias=True, exclude_none=True)
+ except AttributeError:
+ body = request.dict(by_alias=True, exclude_none=True)
+
+ api = Api()
+ data, _ = api.send_request(
+ method=HttpMethods.POST,
+ endpoint=Endpoints.VULNERABILITIES_ENDPOINT,
+ body=body,
+ )
+
+ self.vulnerability_id = data["id"]
+ Console().print(
+ "[rgb(5,245,141)]✓[/rgb(5,245,141)] Vulnerability "
+ f"'{self.name}' uploaded successfully "
+ f"(id: [bold]{self.vulnerability_id}[/bold])"
+ )
+ return self.vulnerability_id
+
+ def pull(self) -> None:
+ api = Api()
+ data, _ = api.send_request(
+ method=HttpMethods.GET,
+ endpoint=Endpoints.VULNERABILITY_ENDPOINT,
+ url_params={"vulnerabilityId": self.name},
+ )
+
+ response = CustomVulnerabilityHttpResponse(**data["vulnerability"])
+
+ if response.built_in:
+ raise ValueError(
+ f"'{response.name}' is a built-in vulnerability, not a custom one. "
+ "Import it from deepteam.vulnerabilities instead of pulling it."
+ )
+ if not response.criteria:
+ raise ValueError(
+ f"Custom vulnerability '{response.name}' has no criteria on Confident AI, "
+ "so there is nothing to red team it against. Set its criteria and pull again."
+ )
+
+ self.name = response.name
+ self.criteria = response.criteria.strip()
+ self.types = _build_types(
+ response.name, [t.name for t in response.vulnerability_types]
+ )
+ self.evaluation_guidelines = response.evaluation_guidelines or None
+ self.evaluation_examples = [
+ EvaluationExample(
+ input=example.input,
+ actual_output=example.actual_output,
+ score=example.score,
+ reason=example.reason,
+ )
+ for example in response.evaluation_examples
+ ] or None
+ self.metric = None
+ self.vulnerability_id = response.id
+
+ Console().print(
+ "[rgb(5,245,141)]✓[/rgb(5,245,141)] Vulnerability '{self.name}' pulled successfully"
+ )