Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

test files addedd for yaml config checks

A-Vamshife79ed43ebb4
diff --git a/tests/test_core/test_yaml/config.yaml b/tests/test_core/test_yaml/config.yaml
new file mode 100644
index 00000000..3255eb23
--- /dev/null
+++ b/tests/test_core/test_yaml/config.yaml
@@ -0,0 +1,24 @@
+# Red teaming models (separate from target)
+models:
+ simulator: gpt-3.5-turbo-0125
+ evaluation: gpt-4o
+
+# Target system configuration
+target:
+ purpose: "A helpful AI assistant for customer support"
+ model:
+ provider: gemini
+ model: gemini-2.5-flash
+
+# System configuration
+system_config:
+ max_concurrent: 10
+ run_async: false
+ ignore_errors: false
+
+default_vulnerabilities:
+ - name: "Bias"
+ types: ["race"]
+
+attacks:
+ - name: "Roleplay"
diff --git a/tests/test_core/test_yaml/config_for_callback.yaml b/tests/test_core/test_yaml/config_for_callback.yaml
new file mode 100644
index 00000000..5d112273
--- /dev/null
+++ b/tests/test_core/test_yaml/config_for_callback.yaml
@@ -0,0 +1,6 @@
+# Red teaming models (separate from target)
+target:
+ purpose: "A custom chatbot"
+ callback:
+ file: "tests/test_core/test_yaml/my_callback.py"
+ function: "model_callback"
\ No newline at end of file
diff --git a/tests/test_core/test_yaml/config_for_class.yaml b/tests/test_core/test_yaml/config_for_class.yaml
new file mode 100644
index 00000000..aeb25925
--- /dev/null
+++ b/tests/test_core/test_yaml/config_for_class.yaml
@@ -0,0 +1,13 @@
+# Red teaming models (separate from target)
+models:
+ simulator:
+ model:
+ provider: custom
+ file: "tests/test_core/test_yaml/my_model.py"
+ class: "CustomModel"
+ evaluation:
+ model:
+ provider: custom
+ file: "tests/test_core/test_yaml/my_model.py"
+ class: "CustomModel"
+
diff --git a/tests/test_core/test_yaml/my_callback.py b/tests/test_core/test_yaml/my_callback.py
new file mode 100644
index 00000000..b8d47474
--- /dev/null
+++ b/tests/test_core/test_yaml/my_callback.py
@@ -0,0 +1,5 @@
+# dummy callback
+
+
+def model_callback(input):
+ return input
diff --git a/tests/test_core/test_yaml/my_model.py b/tests/test_core/test_yaml/my_model.py
new file mode 100644
index 00000000..d6940da2
--- /dev/null
+++ b/tests/test_core/test_yaml/my_model.py
@@ -0,0 +1,20 @@
+from pydantic import BaseModel
+
+from deepeval.models import DeepEvalBaseLLM
+
+
+class CustomModel(DeepEvalBaseLLM):
+ def __init__(self):
+ self.model = "CustomModel"
+
+ def load_model(self):
+ return self.model
+
+ def generate(self, prompt: str, schema: BaseModel) -> BaseModel:
+ return prompt
+
+ async def a_generate(self, prompt: str, schema: BaseModel) -> BaseModel:
+ return self.generate(prompt, schema)
+
+ def get_model_name(self):
+ return "CustomModel"
diff --git a/tests/test_core/test_yaml/test_yaml.py b/tests/test_core/test_yaml/test_yaml.py
new file mode 100644
index 00000000..176c728d
--- /dev/null
+++ b/tests/test_core/test_yaml/test_yaml.py
@@ -0,0 +1,69 @@
+import pytest
+import os
+from deepteam.cli.main import run, _load_config, _load_callback_from_file
+from deepteam.cli.model_callback import load_model
+from deepteam.red_teamer.risk_assessment import RiskAssessment
+from deepeval.models import DeepEvalBaseLLM
+
+
+def test_cli_run_return_both_risk_and_file(tmp_path):
+ OUTPUT_FOLDER = tmp_path / "results_file"
+
+ result = run("config.yaml", 1, 1, str(OUTPUT_FOLDER))
+
+ assert result.risk_assessment is not None
+ assert result.file_path is not None
+ assert isinstance(result.risk_assessment, RiskAssessment)
+ assert str(result.file_path).startswith(str(OUTPUT_FOLDER))
+
+
+def test_cli_run_return_both_risk_and_none():
+ result = run("config.yaml", 1, 1, None)
+
+ assert result.risk_assessment is not None
+ assert result.file_path is None
+ assert isinstance(result.risk_assessment, RiskAssessment)
+
+
+def test_target_loads_from_yaml_callback():
+ TEST = "Check callback returns same @123"
+
+ test_dir = os.path.dirname(__file__)
+ config_path = os.path.join(test_dir, "config_for_callback.yaml")
+ cfg = _load_config(config_path)
+ target_cfg = cfg.get("target", {})
+
+ callback_cfg = target_cfg["callback"]
+ file_path = callback_cfg.get("file")
+ function_name = callback_cfg.get("function")
+
+ model_callback = _load_callback_from_file(file_path, function_name)
+
+ assert callable(model_callback)
+ assert model_callback(TEST) == TEST
+
+
+def test_models_loads_from_yaml_class():
+
+ test_dir = os.path.dirname(__file__)
+ config_path = os.path.join(test_dir, "config_for_class.yaml")
+ cfg = _load_config(config_path)
+ models_cfg = cfg.get("models")
+
+ simulator_model_spec = models_cfg.get("simulator")
+ _simulator_model_spec = simulator_model_spec["model"]
+ simulator_model = load_model(_simulator_model_spec)
+
+ evaluation_model_spec = models_cfg.get("evaluation")
+ _evaluation_model_spec = evaluation_model_spec["model"]
+ evaluation_model = load_model(_evaluation_model_spec)
+
+ assert models_cfg is not None
+ assert (
+ simulator_model_spec is not None and evaluation_model_spec is not None
+ )
+ assert (
+ _simulator_model_spec is not None and _evaluation_model_spec is not None
+ )
+ assert isinstance(simulator_model, DeepEvalBaseLLM)
+ assert isinstance(evaluation_model, DeepEvalBaseLLM)