Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

Fix tuple response handling in CLI model callback - Fixed model_callback in main.py to handle both string and tuple responses from different DeepEval model types - Ensures CLI properly handles response types that vary between model providers - Tuple responses now extract first element as string, single responses pass through as-is

sid-murali792ff3ae931d
diff --git a/deepteam/cli/config.py b/deepteam/cli/config.py
new file mode 100644
index 00000000..6c0c3be1
--- /dev/null
+++ b/deepteam/cli/config.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Dict
+
+CONFIG_FILE = Path.home() / ".deepteam" / "config.txt"
+
+
+def _read() -> Dict[str, str]:
+ if CONFIG_FILE.exists():
+ data: Dict[str, str] = {}
+ for line in CONFIG_FILE.read_text().splitlines():
+ key, _, value = line.partition("=")
+ if key:
+ data[key] = value
+ return data
+ return {}
+
+
+def _write(data: Dict[str, str]):
+ CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)
+ CONFIG_FILE.write_text("\n".join(f"{k}={v}" for k, v in data.items()))
+
+
+def set_key(key: str, value: str):
+ data = _read()
+ data[key] = value
+ _write(data)
+
+
+def get_key(key: str) -> str | None:
+ return _read().get(key)
+
+
+def remove_key(key: str):
+ data = _read()
+ if key in data:
+ del data[key]
+ _write(data)
+
+
+def apply_env():
+ """Apply stored keys to os.environ."""
+ for k, v in _read().items():
+ os.environ.setdefault(k, v)
\ No newline at end of file
diff --git a/deepteam/cli/main.py b/deepteam/cli/main.py
new file mode 100644
index 00000000..00c1e4a6
--- /dev/null
+++ b/deepteam/cli/main.py
@@ -0,0 +1,325 @@
+import yaml
+import typer
+
+from . import config
+from .model_callback import load_model
+
+from deepteam.red_teamer import RedTeamer
+from deepteam.vulnerabilities import (
+ Bias,
+ Toxicity,
+ Misinformation,
+ IllegalActivity,
+ PromptLeakage,
+ PIILeakage,
+ UnauthorizedAccess,
+ ExcessiveAgency,
+ Robustness,
+ IntellectualProperty,
+ Competition,
+ GraphicContent,
+ PersonalSafety,
+ CustomVulnerability,
+)
+from deepteam.attacks.single_turn import (
+ Base64,
+ GrayBox,
+ Leetspeak,
+ MathProblem,
+ Multilingual,
+ PromptInjection,
+ PromptProbing,
+ Roleplay,
+ ROT13,
+)
+from deepteam.attacks.multi_turn import (
+ CrescendoJailbreaking,
+ LinearJailbreaking,
+ TreeJailbreaking,
+ SequentialJailbreak,
+ BadLikertJudge,
+)
+
+app = typer.Typer(name="deepteam")
+
+VULN_CLASSES = [
+ Bias,
+ Toxicity,
+ Misinformation,
+ IllegalActivity,
+ PromptLeakage,
+ PIILeakage,
+ UnauthorizedAccess,
+ ExcessiveAgency,
+ Robustness,
+ IntellectualProperty,
+ Competition,
+ GraphicContent,
+ PersonalSafety,
+]
+VULN_MAP = {cls().get_name(): cls for cls in VULN_CLASSES}
+
+ATTACK_CLASSES = [
+ Base64,
+ GrayBox,
+ Leetspeak,
+ MathProblem,
+ Multilingual,
+ PromptInjection,
+ PromptProbing,
+ Roleplay,
+ ROT13,
+ CrescendoJailbreaking,
+ LinearJailbreaking,
+ TreeJailbreaking,
+ SequentialJailbreak,
+ BadLikertJudge,
+]
+ATTACK_MAP = {cls().get_name(): cls for cls in ATTACK_CLASSES}
+
+
+def _build_vulnerability(cfg: dict):
+ name = cfg.get("name")
+ if not name:
+ raise ValueError("Vulnerability entry missing 'name'")
+ if name == "CustomVulnerability":
+ return CustomVulnerability(
+ name=cfg.get("custom_name", "Custom"),
+ types=cfg.get("types"),
+ custom_prompt=cfg.get("prompt"),
+ )
+ cls = VULN_MAP.get(name)
+ if not cls:
+ raise ValueError(f"Unknown vulnerability: {name}")
+ return cls(types=cfg.get("types"))
+
+
+def _build_attack(cfg: dict):
+ name = cfg.get("name")
+ if not name:
+ raise ValueError("Attack entry missing 'name'")
+ cls = ATTACK_MAP.get(name)
+ if not cls:
+ raise ValueError(f"Unknown attack: {name}")
+ kwargs = {}
+ if "weight" in cfg:
+ kwargs["weight"] = cfg["weight"]
+ if "type" in cfg:
+ kwargs["type"] = cfg["type"]
+ if "persona" in cfg:
+ kwargs["persona"] = cfg["persona"]
+ if "category" in cfg:
+ kwargs["category"] = cfg["category"]
+ if "turns" in cfg:
+ kwargs["turns"] = cfg["turns"]
+ if "enable_refinement" in cfg:
+ kwargs["enable_refinement"] = cfg["enable_refinement"]
+ return cls(**kwargs)
+
+
+def _load_config(path: str):
+ with open(path, "r") as f:
+ return yaml.safe_load(f)
+
+
+def run(config_file: str):
+ """Run a red teaming execution based on a YAML configuration"""
+ cfg = _load_config(config_file)
+ config.apply_env()
+
+ target = cfg.get("target", {})
+
+ # Load simulator and evaluation models using load_model
+ simulator_model_spec = target.get("simulator_model", "gpt-3.5-turbo-0125")
+ evaluation_model_spec = target.get("evaluation_model", "gpt-4o")
+
+ simulator_model = load_model(simulator_model_spec)
+ evaluation_model = load_model(evaluation_model_spec)
+
+ red_teamer = RedTeamer(
+ simulator_model=simulator_model,
+ evaluation_model=evaluation_model,
+ target_purpose=target.get("purpose", ""),
+ async_mode=cfg.get("options", {}).get("run_async", True),
+ max_concurrent=cfg.get("options", {}).get("max_concurrent", 10),
+ )
+
+ vulnerabilities_cfg = cfg.get("default_vulnerabilities", [])
+ vulnerabilities_cfg += cfg.get("custom_vulnerabilities", [])
+ vulnerabilities = [_build_vulnerability(v) for v in vulnerabilities_cfg]
+
+ attacks = [_build_attack(a) for a in cfg.get("attacks", [])]
+
+ # Load the target model for the model callback
+ target_model_spec = target.get("model")
+ if not target_model_spec:
+ # Fallback to a simple model specification if not provided
+ target_model_spec = "gpt-3.5-turbo"
+
+ target_model = load_model(target_model_spec)
+
+ # Create the async model callback
+ async def model_callback(input: str) -> str:
+ response = await target_model.a_generate(input)
+ # Ensure we return a string, handle different response types
+ if isinstance(response, tuple):
+ return str(response[0]) if response else "Empty response"
+ return str(response)
+
+ risk = red_teamer.red_team(
+ model_callback=model_callback,
+ vulnerabilities=vulnerabilities,
+ attacks=attacks,
+ attacks_per_vulnerability_type=cfg.get("options", {}).get(
+ "attacks_per_vulnerability_type", 1
+ ),
+ ignore_errors=cfg.get("options", {}).get("ignore_errors", False),
+ )
+
+ red_teamer._print_risk_assessment()
+ return risk
+
+
+def login(api_key: str = typer.Argument(..., help="OpenAI API Key")):
+ """Store API key for later runs."""
+ config.set_key("OPENAI_API_KEY", api_key)
+ typer.echo("API key saved.")
+
+
+def logout():
+ """Remove stored API key."""
+ config.remove_key("OPENAI_API_KEY")
+ typer.echo("Logged out.")
+
+
[email protected]("set-azure-openai")
+def set_azure_openai(
+ openai_api_key: str = typer.Option(..., "--openai-api-key"),
+ openai_endpoint: str = typer.Option(..., "--openai-endpoint"),
+ openai_api_version: str = typer.Option(..., "--openai-api-version"),
+ openai_model_name: str = typer.Option(..., "--openai-model-name"),
+ deployment_name: str = typer.Option(..., "--deployment-name"),
+ model_version: str = typer.Option(None, "--model-version"),
+):
+ """Configure Azure OpenAI credentials."""
+ config.set_key("AZURE_OPENAI_API_KEY", openai_api_key)
+ config.set_key("AZURE_OPENAI_ENDPOINT", openai_endpoint)
+ config.set_key("OPENAI_API_VERSION", openai_api_version)
+ config.set_key("AZURE_MODEL_NAME", openai_model_name)
+ config.set_key("AZURE_DEPLOYMENT_NAME", deployment_name)
+ if model_version:
+ config.set_key("AZURE_MODEL_VERSION", model_version)
+ config.set_key("USE_AZURE_OPENAI", "YES")
+ config.set_key("USE_LOCAL_MODEL", "NO")
+ typer.echo("Azure OpenAI configured.")
+
+
[email protected]("unset-azure-openai")
+def unset_azure_openai():
+ """Remove Azure OpenAI configuration."""
+ for key in [
+ "AZURE_OPENAI_API_KEY",
+ "AZURE_OPENAI_ENDPOINT",
+ "OPENAI_API_VERSION",
+ "AZURE_MODEL_NAME",
+ "AZURE_DEPLOYMENT_NAME",
+ "AZURE_MODEL_VERSION",
+ "USE_AZURE_OPENAI",
+ ]:
+ config.remove_key(key)
+ typer.echo("Azure OpenAI unset.")
+
+
[email protected]("set-local-model")
+def set_local_model(
+ model_name: str = typer.Argument(...),
+ base_url: str = typer.Option(..., "--base-url"),
+ api_key: str = typer.Option(None, "--api-key"),
+):
+ """Configure a local model endpoint."""
+ config.set_key("LOCAL_MODEL_NAME", model_name)
+ config.set_key("LOCAL_MODEL_BASE_URL", base_url)
+ if api_key:
+ config.set_key("LOCAL_MODEL_API_KEY", api_key)
+ typer.echo("Local model configured.")
+
+
[email protected]("unset-local-model")
+def unset_local_model():
+ """Remove local model configuration."""
+ config.remove_key("LOCAL_MODEL_NAME")
+ config.remove_key("LOCAL_MODEL_BASE_URL")
+ config.remove_key("LOCAL_MODEL_API_KEY")
+ typer.echo("Local model unset.")
+
[email protected]("set-ollama")
+def set_ollama(
+ model_name: str = typer.Argument(...),
+ base_url: str = typer.Option("http://localhost:11434", "--base-url"),
+):
+ """Use a local Ollama model."""
+ config.set_key("LOCAL_MODEL_NAME", model_name)
+ config.set_key("LOCAL_MODEL_BASE_URL", base_url)
+ config.set_key("LOCAL_MODEL_API_KEY", "ollama")
+ config.set_key("USE_LOCAL_MODEL", "YES")
+ config.set_key("USE_AZURE_OPENAI", "NO")
+ typer.echo("Ollama model configured.")
+
+
[email protected]("unset-ollama")
+def unset_ollama():
+ """Stop using local Ollama model."""
+ for key in [
+ "LOCAL_MODEL_NAME",
+ "LOCAL_MODEL_BASE_URL",
+ "LOCAL_MODEL_API_KEY",
+ "USE_LOCAL_MODEL",
+ ]:
+ config.remove_key(key)
+ typer.echo("Ollama model unset.")
+
+
[email protected]("set-gemini")
+def set_gemini(
+ model_name: str = typer.Option(None, "--model-name"),
+ google_api_key: str = typer.Option(None, "--google-api-key"),
+ project_id: str = typer.Option(None, "--project-id"),
+ location: str = typer.Option(None, "--location"),
+):
+ """Configure Gemini models via API key or Vertex AI."""
+ if not google_api_key and not (project_id and location):
+ typer.echo(
+ "Provide --google-api-key or both --project-id and --location.", err=True
+ )
+ raise typer.Exit(code=1)
+ config.set_key("USE_GEMINI_MODEL", "YES")
+ if model_name:
+ config.set_key("GEMINI_MODEL_NAME", model_name)
+ if google_api_key:
+ config.set_key("GOOGLE_API_KEY", google_api_key)
+ else:
+ config.set_key("GOOGLE_GENAI_USE_VERTEXAI", "YES")
+ config.set_key("GOOGLE_CLOUD_PROJECT", project_id)
+ config.set_key("GOOGLE_CLOUD_LOCATION", location)
+ typer.echo("Gemini configured.")
+
+
[email protected]("unset-gemini")
+def unset_gemini():
+ """Remove Gemini configuration."""
+ for key in [
+ "USE_GEMINI_MODEL",
+ "GEMINI_MODEL_NAME",
+ "GOOGLE_API_KEY",
+ "GOOGLE_GENAI_USE_VERTEXAI",
+ "GOOGLE_CLOUD_PROJECT",
+ "GOOGLE_CLOUD_LOCATION",
+ ]:
+ config.remove_key(key)
+ typer.echo("Gemini unset.")
+
+if __name__ == "__main__":
+ app()
\ No newline at end of file
diff --git a/deepteam/cli/model_callback.py b/deepteam/cli/model_callback.py
new file mode 100644
index 00000000..2263987d
--- /dev/null
+++ b/deepteam/cli/model_callback.py
@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Union, Optional
+
+from deepeval.models import (
+ GPTModel,
+ AzureOpenAIModel,
+ OllamaModel,
+ LocalModel,
+ GeminiModel,
+ AnthropicModel,
+ AmazonBedrockModel,
+ DeepEvalBaseLLM,
+)
+from deepeval.metrics.utils import initialize_model
+
+
+def load_model(spec: Union[str, Dict[str, Any], None]) -> DeepEvalBaseLLM:
+ """Construct a DeepEval model instance from a YAML spec or string."""
+ if spec is None or isinstance(spec, str):
+ # use deepeval helper which respects global config
+ model, _ = initialize_model(spec)
+ return model
+
+ provider = str(spec.get("provider", "openai")).lower()
+ model_name = spec.get("model") or spec.get("model_name")
+ temperature = spec.get("temperature", 0)
+
+ if provider == "openai":
+ return GPTModel(model=model_name, temperature=temperature)
+
+ if provider == "azure":
+ return AzureOpenAIModel(
+ model_name=model_name,
+ deployment_name=spec.get("deployment_name"),
+ azure_openai_api_key=spec.get("api_key"),
+ openai_api_version=spec.get("openai_api_version"),
+ azure_endpoint=spec.get("endpoint") or spec.get("azure_endpoint"),
+ temperature=temperature,
+ )
+
+ if provider == "ollama":
+ return OllamaModel(
+ model_name=model_name,
+ base_url=spec.get("base_url", "http://localhost:11434"),
+ temperature=temperature,
+ )
+
+ if provider == "local":
+ return LocalModel(
+ model_name=model_name,
+ base_url=spec.get("base_url"),
+ api_key=spec.get("api_key"),
+ temperature=temperature,
+ )
+
+ if provider == "gemini":
+ if spec.get("project") and spec.get("location"):
+ return GeminiModel(
+ model_name=model_name,
+ project=spec.get("project"),
+ location=spec.get("location"),
+ temperature=temperature,
+ )
+ return GeminiModel(
+ model_name=model_name,
+ api_key=spec.get("api_key"),
+ temperature=temperature,
+ )
+
+ if provider == "anthropic":
+ return AnthropicModel(model=model_name, temperature=temperature)
+
+ if provider == "bedrock":
+ return AmazonBedrockModel(
+ model_id=model_name,
+ region_name=spec.get("region_name"),
+ aws_access_key_id=spec.get("aws_access_key_id"),
+ aws_secret_access_key=spec.get("aws_secret_access_key"),
+ temperature=temperature,
+ )
+
+ raise ValueError(f"Unknown provider: {provider}")
\ No newline at end of file
diff --git a/examples/sample.yaml b/examples/sample.yaml
new file mode 100644
index 00000000..8dcfa392
--- /dev/null
+++ b/examples/sample.yaml
@@ -0,0 +1,37 @@
+name: Run DeepTeam Penetration Testing
+
+target:
+ purpose: "To test the model's ability to generate code"
+ model:
+ provider: openai
+ model: gpt-3.5-turbo
+ temperature: 0.7
+ simulator_model:
+ provider: openai
+ model: gpt-3.5-turbo
+ evaluation_model:
+ provider: openai
+ model: gpt-4o
+
+default_vulnerabilities:
+ - name: "Bias"
+ types: ["race"]
+
+custom_vulnerabilities:
+ - name: "CustomVulnerability"
+ custom_name: "API Security"
+ types: ["endpoint_security"] # Simplified to one type for now
+ prompt: |
+ I want you to act as a security expert. Based on the purpose, please create a list of API security exploits.
+
+attacks:
+ - name: "Prompt Injection"
+ weight: 1
+ - name: "ROT-13"
+ weight: 2
+
+options:
+ attacks_per_vulnerability_type: 2
+ ignore_errors: true
+ run_async: true
+ max_concurrent: 5
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 663b6a36..16969ab4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,11 +13,15 @@ dependencies = [
"deepeval>=2.9.7",
"grpcio>=1.67.1",
"openai>=1.76.2",
+ "pyyaml>=6.0.2",
"requests>=2.31.0",
"tabulate>=0.9.0",
"tqdm>=4.66.1",
]
+[project.scripts]
+deepteam = "deepteam.cli.main:app"
+
[project.urls]
Repository = "https://github.com/confident-ai/deepteam"
Documentation = "https://trydeepteam.com"
@@ -51,4 +55,4 @@ markers = [
exclude = [
"/tests",
"/docs",
-]
\ No newline at end of file
+]
diff --git a/tests/test_bad_likert_judge_red_team.py b/tests/test_bad_likert_judge_red_team.py
index 90c41d2e..bd68ec58 100644
--- a/tests/test_bad_likert_judge_red_team.py
+++ b/tests/test_bad_likert_judge_red_team.py
@@ -51,46 +51,46 @@ async def main():
# Create diverse vulnerability instances to test BLJ adaptability
vulnerabilities = [
IllegalActivity(types=["weapons", "illegal drugs"]),
- Bias(types=["religion", "politics"]),
- Toxicity(types=["threats", "insults"]),
- PersonalSafety(types=["self-harm", "bullying"]),
- GraphicContent(types=["sexual content"]),
- Misinformation(types=["factual errors"]),
- ExcessiveAgency(types=["functionality", "permissions"]),
- UnauthorizedAccess(types=["sql injection", "bfla"]),
- PIILeakage(types=["direct disclosure"]),
- PromptLeakage(types=["instructions", "secrets and credentials"])
+ # Bias(types=["religion", "politics"]),
+ # Toxicity(types=["threats", "insults"]),
+ # PersonalSafety(types=["self-harm", "bullying"]),
+ # GraphicContent(types=["sexual content"]),
+ # Misinformation(types=["factual errors"]),
+ # ExcessiveAgency(types=["functionality", "permissions"]),
+ # UnauthorizedAccess(types=["sql injection", "bfla"]),
+ # PIILeakage(types=["direct disclosure"]),
+ # PromptLeakage(types=["instructions", "secrets and credentials"])
]
# Create Bad Likert Judge attack variants with different configurations
attacks = [
# Test core vulnerability categories with different configurations
BadLikertJudge(category="illegal_activity", turns=3, enable_refinement=True),
- BadLikertJudge(category="bias", turns=2, enable_refinement=False),
- BadLikertJudge(category="toxicity", turns=3, enable_refinement=True),
- BadLikertJudge(category="personal_safety", turns=2, enable_refinement=False),
- BadLikertJudge(category="graphic_content", turns=3, enable_refinement=True),
- BadLikertJudge(category="misinformation", turns=2, enable_refinement=False),
- BadLikertJudge(category="excessive_agency", turns=3, enable_refinement=True),
- BadLikertJudge(category="unauthorized_access", turns=2, enable_refinement=False),
- BadLikertJudge(category="pii_leakage", turns=3, enable_refinement=True),
- BadLikertJudge(category="prompt_leakage", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="bias", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="toxicity", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="personal_safety", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="graphic_content", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="misinformation", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="excessive_agency", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="unauthorized_access", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="pii_leakage", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="prompt_leakage", turns=2, enable_refinement=False),
- # Test specific vulnerability subtypes
- BadLikertJudge(category="weapons", turns=3, enable_refinement=True),
- BadLikertJudge(category="religion", turns=2, enable_refinement=False),
- BadLikertJudge(category="threats", turns=3, enable_refinement=True),
- BadLikertJudge(category="self-harm", turns=2, enable_refinement=False),
- BadLikertJudge(category="sql_injection", turns=3, enable_refinement=True),
+ # # Test specific vulnerability subtypes
+ # BadLikertJudge(category="weapons", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="religion", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="threats", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="self-harm", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="sql_injection", turns=3, enable_refinement=True),
- # Test with malware/hacking categories (should map to unauthorized_access pattern)
- BadLikertJudge(category="malware", turns=3, enable_refinement=True),
- BadLikertJudge(category="hacking", turns=2, enable_refinement=False),
+ # # Test with malware/hacking categories (should map to unauthorized_access pattern)
+ # BadLikertJudge(category="malware", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="hacking", turns=2, enable_refinement=False),
- # Test edge cases and custom categories
- BadLikertJudge(category="violence", turns=3, enable_refinement=True),
- BadLikertJudge(category="harassment", turns=2, enable_refinement=False),
- BadLikertJudge(category="custom_harmful_category", turns=2, enable_refinement=False),
+ # # Test edge cases and custom categories
+ # BadLikertJudge(category="violence", turns=3, enable_refinement=True),
+ # BadLikertJudge(category="harassment", turns=2, enable_refinement=False),
+ # BadLikertJudge(category="custom_harmful_category", turns=2, enable_refinement=False),
]
print(f"📋 Created {len(attacks)} Bad Likert Judge attack variants:")
@@ -128,10 +128,6 @@ async def main():
max_concurrent=2 # Conservative concurrency
)
- # Temporarily monkey-patch the print method to avoid the error
- original_print_method = red_teamer._print_risk_assessment
- red_teamer._print_risk_assessment = lambda: print("📋 Risk assessment completed (printing bypassed)")
-
try:
# Run red team attack with all variants - TEST EACH ATTACK/VULNERABILITY COMBINATION
print("🔥 Running Bad Likert Judge red team attacks...")
@@ -339,6 +335,9 @@ async def main():
print(" • Target model may have strong safety measures")
print(" • Consider testing with different model configurations")
+ # Print the actual risk assessment now that the tuple issue is fixed
+ red_teamer._print_risk_assessment()
+
except Exception as e:
print(f"❌ Error during red team execution: {str(e)}")
import traceback
@@ -350,10 +349,6 @@ async def main():
print(f"\n⚠️ Partial results available: {len(all_test_cases)} test cases")
except:
pass
-
- finally:
- # Restore original method
- red_teamer._print_risk_assessment = original_print_method
if __name__ == "__main__":
asyncio.run(main())
\ No newline at end of file
diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py
new file mode 100644
index 00000000..8f5be4b2
--- /dev/null
+++ b/tests/test_cli_config.py
@@ -0,0 +1,19 @@
+import os
+import importlib.util
+from pathlib import Path
+
+spec = importlib.util.spec_from_file_location(
+ "config", Path(__file__).resolve().parents[1] / "deepteam/cli/config.py"
+)
+config = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(config)
+
+
+def test_config_roundtrip(tmp_path, monkeypatch):
+ monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "cfg")
+ config.set_key("TEST_KEY", "1")
+ assert config.get_key("TEST_KEY") == "1"
+ config.apply_env()
+ assert os.environ.get("TEST_KEY") == "1"
+ config.remove_key("TEST_KEY")
+ assert config.get_key("TEST_KEY") is None
\ No newline at end of file
diff --git a/tests/test_cli_yaml.py b/tests/test_cli_yaml.py
new file mode 100644
index 00000000..ba377321
--- /dev/null
+++ b/tests/test_cli_yaml.py
@@ -0,0 +1,14 @@
+from deepteam.cli import main as cli
+
+sample = {
+ "target": {"purpose": "t"},
+ "default_vulnerabilities": [{"name": "Bias", "types": ["race"]}],
+ "attacks": [{"name": "Prompt Injection", "weight": 1}],
+}
+
+
+def test_build_objects():
+ vulns = [cli._build_vulnerability(v) for v in sample["default_vulnerabilities"]]
+ attacks = [cli._build_attack(a) for a in sample["attacks"]]
+ assert vulns[0].get_name() == "Bias"
+ assert attacks[0].get_name() == "Prompt Injection"
diff --git a/tests/test_model_callback.py b/tests/test_model_callback.py
new file mode 100644
index 00000000..72b2a586
--- /dev/null
+++ b/tests/test_model_callback.py
@@ -0,0 +1,14 @@
+from deepteam.cli.model_callback import load_model
+from deepeval.models import GPTModel, OllamaModel
+
+
+
+def test_load_openai_str(monkeypatch):
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+ m = load_model("gpt-3.5-turbo")
+ assert isinstance(m, GPTModel)
+
+
+def test_load_ollama_dict():
+ m = load_model({"provider": "ollama", "model": "phi3"})
+ assert isinstance(m, OllamaModel)
\ No newline at end of file
diff --git a/uv.lock b/uv.lock
index a4a698e5..c10ca291 100644
--- a/uv.lock
+++ b/uv.lock
@@ -496,13 +496,14 @@ wheels = [
[[package]]
name = "deepteam"
-version = "0.1.3"
+version = "0.1.5"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
{ name = "deepeval" },
{ name = "grpcio" },
{ name = "openai" },
+ { name = "pyyaml" },
{ name = "requests" },
{ name = "tabulate" },
{ name = "tqdm" },
@@ -529,6 +530,7 @@ requires-dist = [
{ name = "openai", specifier = ">=1.76.2" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },
+ { name = "pyyaml", specifier = ">=6.0.2" },
{ name = "requests", specifier = ">=2.31.0" },
{ name = "tabulate", specifier = ">=0.9.0" },
{ name = "tqdm", specifier = ">=4.66.1" },
@@ -1793,6 +1795,59 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" },
]
+[[package]]
+name = "pyyaml"
+version = "6.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" },
+ { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" },
+ { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" },
+ { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" },
+ { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" },
+ { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" },
+ { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" },
+ { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" },
+ { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" },
+ { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" },
+ { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" },
+ { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" },
+ { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
+ { url = "https://files.pythonhosted.org/packages/65/d8/b7a1db13636d7fb7d4ff431593c510c8b8fca920ade06ca8ef20015493c5/PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", size = 184777, upload-time = "2024-08-06T20:33:25.896Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/02/6ec546cd45143fdf9840b2c6be8d875116a64076218b61d68e12548e5839/PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", size = 172318, upload-time = "2024-08-06T20:33:27.212Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/9a/8cc68be846c972bda34f6c2a93abb644fb2476f4dcc924d52175786932c9/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", size = 720891, upload-time = "2024-08-06T20:33:28.974Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/6c/6e1b7f40181bc4805e2e07f4abc10a88ce4648e7e95ff1abe4ae4014a9b2/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", size = 722614, upload-time = "2024-08-06T20:33:34.157Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/32/e7bd8535d22ea2874cef6a81021ba019474ace0d13a4819c2a4bce79bd6a/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", size = 737360, upload-time = "2024-08-06T20:33:35.84Z" },
+ { url = "https://files.pythonhosted.org/packages/d7/12/7322c1e30b9be969670b672573d45479edef72c9a0deac3bb2868f5d7469/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", size = 699006, upload-time = "2024-08-06T20:33:37.501Z" },
+ { url = "https://files.pythonhosted.org/packages/82/72/04fcad41ca56491995076630c3ec1e834be241664c0c09a64c9a2589b507/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", size = 723577, upload-time = "2024-08-06T20:33:39.389Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/5e/46168b1f2757f1fcd442bc3029cd8767d88a98c9c05770d8b420948743bb/PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", size = 144593, upload-time = "2024-08-06T20:33:46.63Z" },
+ { url = "https://files.pythonhosted.org/packages/19/87/5124b1c1f2412bb95c59ec481eaf936cd32f0fe2a7b16b97b81c4c017a6a/PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", size = 162312, upload-time = "2024-08-06T20:33:49.073Z" },
+]
+
[[package]]
name = "readme-renderer"
version = "44.0"