DeepTeam is a framework to red team LLMs and AI agents.
Fix tuple response handling in CLI model callback - Fixed model_callback in main.py to handle both string and tuple responses from different DeepEval model types - Ensures CLI properly handles response types that vary between model providers - Tuple responses now extract first element as string, single responses pass through as-is
sid-murali792ff3ae931d
diff --git a/deepteam/cli/config.py b/deepteam/cli/config.pynew file mode 100644index 00000000..6c0c3be1--- /dev/null+++ b/deepteam/cli/config.py@@ -0,0 +1,46 @@+from __future__ import annotations++import os+from pathlib import Path+from typing import Dict++CONFIG_FILE = Path.home() / ".deepteam" / "config.txt"+++def _read() -> Dict[str, str]:+ if CONFIG_FILE.exists():+ data: Dict[str, str] = {}+ for line in CONFIG_FILE.read_text().splitlines():+ key, _, value = line.partition("=")+ if key:+ data[key] = value+ return data+ return {}+++def _write(data: Dict[str, str]):+ CONFIG_FILE.parent.mkdir(parents=True, exist_ok=True)+ CONFIG_FILE.write_text("\n".join(f"{k}={v}" for k, v in data.items()))+++def set_key(key: str, value: str):+ data = _read()+ data[key] = value+ _write(data)+++def get_key(key: str) -> str | None:+ return _read().get(key)+++def remove_key(key: str):+ data = _read()+ if key in data:+ del data[key]+ _write(data)+++def apply_env():+ """Apply stored keys to os.environ."""+ for k, v in _read().items():+ os.environ.setdefault(k, v)\ No newline at end of filediff --git a/deepteam/cli/main.py b/deepteam/cli/main.pynew file mode 100644index 00000000..00c1e4a6--- /dev/null+++ b/deepteam/cli/main.py@@ -0,0 +1,325 @@+import yaml+import typer++from . import config+from .model_callback import load_model++from deepteam.red_teamer import RedTeamer+from deepteam.vulnerabilities import (+ Bias,+ Toxicity,+ Misinformation,+ IllegalActivity,+ PromptLeakage,+ PIILeakage,+ UnauthorizedAccess,+ ExcessiveAgency,+ Robustness,+ IntellectualProperty,+ Competition,+ GraphicContent,+ PersonalSafety,+ CustomVulnerability,+)+from deepteam.attacks.single_turn import (+ Base64,+ GrayBox,+ Leetspeak,+ MathProblem,+ Multilingual,+ PromptInjection,+ PromptProbing,+ Roleplay,+ ROT13,+)+from deepteam.attacks.multi_turn import (+ CrescendoJailbreaking,+ LinearJailbreaking,+ TreeJailbreaking,+ SequentialJailbreak,+ BadLikertJudge,+)++app = typer.Typer(name="deepteam")++VULN_CLASSES = [+ Bias,+ Toxicity,+ Misinformation,+ IllegalActivity,+ PromptLeakage,+ PIILeakage,+ UnauthorizedAccess,+ ExcessiveAgency,+ Robustness,+ IntellectualProperty,+ Competition,+ GraphicContent,+ PersonalSafety,+]+VULN_MAP = {cls().get_name(): cls for cls in VULN_CLASSES}++ATTACK_CLASSES = [+ Base64,+ GrayBox,+ Leetspeak,+ MathProblem,+ Multilingual,+ PromptInjection,+ PromptProbing,+ Roleplay,+ ROT13,+ CrescendoJailbreaking,+ LinearJailbreaking,+ TreeJailbreaking,+ SequentialJailbreak,+ BadLikertJudge,+]+ATTACK_MAP = {cls().get_name(): cls for cls in ATTACK_CLASSES}+++def _build_vulnerability(cfg: dict):+ name = cfg.get("name")+ if not name:+ raise ValueError("Vulnerability entry missing 'name'")+ if name == "CustomVulnerability":+ return CustomVulnerability(+ name=cfg.get("custom_name", "Custom"),+ types=cfg.get("types"),+ custom_prompt=cfg.get("prompt"),+ )+ cls = VULN_MAP.get(name)+ if not cls:+ raise ValueError(f"Unknown vulnerability: {name}")+ return cls(types=cfg.get("types"))+++def _build_attack(cfg: dict):+ name = cfg.get("name")+ if not name:+ raise ValueError("Attack entry missing 'name'")+ cls = ATTACK_MAP.get(name)+ if not cls:+ raise ValueError(f"Unknown attack: {name}")+ kwargs = {}+ if "weight" in cfg:+ kwargs["weight"] = cfg["weight"]+ if "type" in cfg:+ kwargs["type"] = cfg["type"]+ if "persona" in cfg:+ kwargs["persona"] = cfg["persona"]+ if "category" in cfg:+ kwargs["category"] = cfg["category"]+ if "turns" in cfg:+ kwargs["turns"] = cfg["turns"]+ if "enable_refinement" in cfg:+ kwargs["enable_refinement"] = cfg["enable_refinement"]+ return cls(**kwargs)+++def _load_config(path: str):+ with open(path, "r") as f:+ return yaml.safe_load(f)+++def run(config_file: str):+ """Run a red teaming execution based on a YAML configuration"""+ cfg = _load_config(config_file)+ config.apply_env()++ target = cfg.get("target", {})++ # Load simulator and evaluation models using load_model+ simulator_model_spec = target.get("simulator_model", "gpt-3.5-turbo-0125")+ evaluation_model_spec = target.get("evaluation_model", "gpt-4o")++ simulator_model = load_model(simulator_model_spec)+ evaluation_model = load_model(evaluation_model_spec)++ red_teamer = RedTeamer(+ simulator_model=simulator_model,+ evaluation_model=evaluation_model,+ target_purpose=target.get("purpose", ""),+ async_mode=cfg.get("options", {}).get("run_async", True),+ max_concurrent=cfg.get("options", {}).get("max_concurrent", 10),+ )++ vulnerabilities_cfg = cfg.get("default_vulnerabilities", [])+ vulnerabilities_cfg += cfg.get("custom_vulnerabilities", [])+ vulnerabilities = [_build_vulnerability(v) for v in vulnerabilities_cfg]++ attacks = [_build_attack(a) for a in cfg.get("attacks", [])]++ # Load the target model for the model callback+ target_model_spec = target.get("model")+ if not target_model_spec:+ # Fallback to a simple model specification if not provided+ target_model_spec = "gpt-3.5-turbo"++ target_model = load_model(target_model_spec)++ # Create the async model callback+ async def model_callback(input: str) -> str:+ response = await target_model.a_generate(input)+ # Ensure we return a string, handle different response types+ if isinstance(response, tuple):+ return str(response[0]) if response else "Empty response"+ return str(response)++ risk = red_teamer.red_team(+ model_callback=model_callback,+ vulnerabilities=vulnerabilities,+ attacks=attacks,+ attacks_per_vulnerability_type=cfg.get("options", {}).get(+ "attacks_per_vulnerability_type", 1+ ),+ ignore_errors=cfg.get("options", {}).get("ignore_errors", False),+ )++ red_teamer._print_risk_assessment()+ return risk+++def login(api_key: str = typer.Argument(..., help="OpenAI API Key")):+ """Store API key for later runs."""+ config.set_key("OPENAI_API_KEY", api_key)+ typer.echo("API key saved.")+++def logout():+ """Remove stored API key."""+ config.remove_key("OPENAI_API_KEY")+ typer.echo("Logged out.")++[email protected]("set-azure-openai")+def set_azure_openai(+ openai_api_key: str = typer.Option(..., "--openai-api-key"),+ openai_endpoint: str = typer.Option(..., "--openai-endpoint"),+ openai_api_version: str = typer.Option(..., "--openai-api-version"),+ openai_model_name: str = typer.Option(..., "--openai-model-name"),+ deployment_name: str = typer.Option(..., "--deployment-name"),+ model_version: str = typer.Option(None, "--model-version"),+):+ """Configure Azure OpenAI credentials."""+ config.set_key("AZURE_OPENAI_API_KEY", openai_api_key)+ config.set_key("AZURE_OPENAI_ENDPOINT", openai_endpoint)+ config.set_key("OPENAI_API_VERSION", openai_api_version)+ config.set_key("AZURE_MODEL_NAME", openai_model_name)+ config.set_key("AZURE_DEPLOYMENT_NAME", deployment_name)+ if model_version:+ config.set_key("AZURE_MODEL_VERSION", model_version)+ config.set_key("USE_AZURE_OPENAI", "YES")+ config.set_key("USE_LOCAL_MODEL", "NO")+ typer.echo("Azure OpenAI configured.")++[email protected]("unset-azure-openai")+def unset_azure_openai():+ """Remove Azure OpenAI configuration."""+ for key in [+ "AZURE_OPENAI_API_KEY",+ "AZURE_OPENAI_ENDPOINT",+ "OPENAI_API_VERSION",+ "AZURE_MODEL_NAME",+ "AZURE_DEPLOYMENT_NAME",+ "AZURE_MODEL_VERSION",+ "USE_AZURE_OPENAI",+ ]:+ config.remove_key(key)+ typer.echo("Azure OpenAI unset.")++[email protected]("set-local-model")+def set_local_model(+ model_name: str = typer.Argument(...),+ base_url: str = typer.Option(..., "--base-url"),+ api_key: str = typer.Option(None, "--api-key"),+):+ """Configure a local model endpoint."""+ config.set_key("LOCAL_MODEL_NAME", model_name)+ config.set_key("LOCAL_MODEL_BASE_URL", base_url)+ if api_key:+ config.set_key("LOCAL_MODEL_API_KEY", api_key)+ typer.echo("Local model configured.")++[email protected]("unset-local-model")+def unset_local_model():+ """Remove local model configuration."""+ config.remove_key("LOCAL_MODEL_NAME")+ config.remove_key("LOCAL_MODEL_BASE_URL")+ config.remove_key("LOCAL_MODEL_API_KEY")+ typer.echo("Local model unset.")+[email protected]("set-ollama")+def set_ollama(+ model_name: str = typer.Argument(...),+ base_url: str = typer.Option("http://localhost:11434", "--base-url"),+):+ """Use a local Ollama model."""+ config.set_key("LOCAL_MODEL_NAME", model_name)+ config.set_key("LOCAL_MODEL_BASE_URL", base_url)+ config.set_key("LOCAL_MODEL_API_KEY", "ollama")+ config.set_key("USE_LOCAL_MODEL", "YES")+ config.set_key("USE_AZURE_OPENAI", "NO")+ typer.echo("Ollama model configured.")++[email protected]("unset-ollama")+def unset_ollama():+ """Stop using local Ollama model."""+ for key in [+ "LOCAL_MODEL_NAME",+ "LOCAL_MODEL_BASE_URL",+ "LOCAL_MODEL_API_KEY",+ "USE_LOCAL_MODEL",+ ]:+ config.remove_key(key)+ typer.echo("Ollama model unset.")++[email protected]("set-gemini")+def set_gemini(+ model_name: str = typer.Option(None, "--model-name"),+ google_api_key: str = typer.Option(None, "--google-api-key"),+ project_id: str = typer.Option(None, "--project-id"),+ location: str = typer.Option(None, "--location"),+):+ """Configure Gemini models via API key or Vertex AI."""+ if not google_api_key and not (project_id and location):+ typer.echo(+ "Provide --google-api-key or both --project-id and --location.", err=True+ )+ raise typer.Exit(code=1)+ config.set_key("USE_GEMINI_MODEL", "YES")+ if model_name:+ config.set_key("GEMINI_MODEL_NAME", model_name)+ if google_api_key:+ config.set_key("GOOGLE_API_KEY", google_api_key)+ else:+ config.set_key("GOOGLE_GENAI_USE_VERTEXAI", "YES")+ config.set_key("GOOGLE_CLOUD_PROJECT", project_id)+ config.set_key("GOOGLE_CLOUD_LOCATION", location)+ typer.echo("Gemini configured.")++[email protected]("unset-gemini")+def unset_gemini():+ """Remove Gemini configuration."""+ for key in [+ "USE_GEMINI_MODEL",+ "GEMINI_MODEL_NAME",+ "GOOGLE_API_KEY",+ "GOOGLE_GENAI_USE_VERTEXAI",+ "GOOGLE_CLOUD_PROJECT",+ "GOOGLE_CLOUD_LOCATION",+ ]:+ config.remove_key(key)+ typer.echo("Gemini unset.")++if __name__ == "__main__":+ app()\ No newline at end of filediff --git a/deepteam/cli/model_callback.py b/deepteam/cli/model_callback.pynew file mode 100644index 00000000..2263987d--- /dev/null+++ b/deepteam/cli/model_callback.py@@ -0,0 +1,83 @@+from __future__ import annotations++from typing import Any, Dict, Union, Optional++from deepeval.models import (+ GPTModel,+ AzureOpenAIModel,+ OllamaModel,+ LocalModel,+ GeminiModel,+ AnthropicModel,+ AmazonBedrockModel,+ DeepEvalBaseLLM,+)+from deepeval.metrics.utils import initialize_model+++def load_model(spec: Union[str, Dict[str, Any], None]) -> DeepEvalBaseLLM:+ """Construct a DeepEval model instance from a YAML spec or string."""+ if spec is None or isinstance(spec, str):+ # use deepeval helper which respects global config+ model, _ = initialize_model(spec)+ return model++ provider = str(spec.get("provider", "openai")).lower()+ model_name = spec.get("model") or spec.get("model_name")+ temperature = spec.get("temperature", 0)++ if provider == "openai":+ return GPTModel(model=model_name, temperature=temperature)++ if provider == "azure":+ return AzureOpenAIModel(+ model_name=model_name,+ deployment_name=spec.get("deployment_name"),+ azure_openai_api_key=spec.get("api_key"),+ openai_api_version=spec.get("openai_api_version"),+ azure_endpoint=spec.get("endpoint") or spec.get("azure_endpoint"),+ temperature=temperature,+ )++ if provider == "ollama":+ return OllamaModel(+ model_name=model_name,+ base_url=spec.get("base_url", "http://localhost:11434"),+ temperature=temperature,+ )++ if provider == "local":+ return LocalModel(+ model_name=model_name,+ base_url=spec.get("base_url"),+ api_key=spec.get("api_key"),+ temperature=temperature,+ )++ if provider == "gemini":+ if spec.get("project") and spec.get("location"):+ return GeminiModel(+ model_name=model_name,+ project=spec.get("project"),+ location=spec.get("location"),+ temperature=temperature,+ )+ return GeminiModel(+ model_name=model_name,+ api_key=spec.get("api_key"),+ temperature=temperature,+ )++ if provider == "anthropic":+ return AnthropicModel(model=model_name, temperature=temperature)++ if provider == "bedrock":+ return AmazonBedrockModel(+ model_id=model_name,+ region_name=spec.get("region_name"),+ aws_access_key_id=spec.get("aws_access_key_id"),+ aws_secret_access_key=spec.get("aws_secret_access_key"),+ temperature=temperature,+ )++ raise ValueError(f"Unknown provider: {provider}")\ No newline at end of filediff --git a/examples/sample.yaml b/examples/sample.yamlnew file mode 100644index 00000000..8dcfa392--- /dev/null+++ b/examples/sample.yaml@@ -0,0 +1,37 @@+name: Run DeepTeam Penetration Testing++target:+ purpose: "To test the model's ability to generate code"+ model:+ provider: openai+ model: gpt-3.5-turbo+ temperature: 0.7+ simulator_model:+ provider: openai+ model: gpt-3.5-turbo+ evaluation_model:+ provider: openai+ model: gpt-4o++default_vulnerabilities:+ - name: "Bias"+ types: ["race"]++custom_vulnerabilities:+ - name: "CustomVulnerability"+ custom_name: "API Security"+ types: ["endpoint_security"] # Simplified to one type for now+ prompt: |+ I want you to act as a security expert. Based on the purpose, please create a list of API security exploits.++attacks:+ - name: "Prompt Injection"+ weight: 1+ - name: "ROT-13"+ weight: 2++options:+ attacks_per_vulnerability_type: 2+ ignore_errors: true+ run_async: true+ max_concurrent: 5\ No newline at end of filediff --git a/pyproject.toml b/pyproject.tomlindex 663b6a36..16969ab4 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -13,11 +13,15 @@ dependencies = ["deepeval>=2.9.7","grpcio>=1.67.1","openai>=1.76.2",+ "pyyaml>=6.0.2","requests>=2.31.0","tabulate>=0.9.0","tqdm>=4.66.1",]+[project.scripts]+deepteam = "deepteam.cli.main:app"+[project.urls]Repository = "https://github.com/confident-ai/deepteam"Documentation = "https://trydeepteam.com"@@ -51,4 +55,4 @@ markers = [exclude = ["/tests","/docs",-]\ No newline at end of file+]diff --git a/tests/test_bad_likert_judge_red_team.py b/tests/test_bad_likert_judge_red_team.pyindex 90c41d2e..bd68ec58 100644--- a/tests/test_bad_likert_judge_red_team.py+++ b/tests/test_bad_likert_judge_red_team.py@@ -51,46 +51,46 @@ async def main():# Create diverse vulnerability instances to test BLJ adaptabilityvulnerabilities = [IllegalActivity(types=["weapons", "illegal drugs"]),- Bias(types=["religion", "politics"]),- Toxicity(types=["threats", "insults"]),- PersonalSafety(types=["self-harm", "bullying"]),- GraphicContent(types=["sexual content"]),- Misinformation(types=["factual errors"]),- ExcessiveAgency(types=["functionality", "permissions"]),- UnauthorizedAccess(types=["sql injection", "bfla"]),- PIILeakage(types=["direct disclosure"]),- PromptLeakage(types=["instructions", "secrets and credentials"])+ # Bias(types=["religion", "politics"]),+ # Toxicity(types=["threats", "insults"]),+ # PersonalSafety(types=["self-harm", "bullying"]),+ # GraphicContent(types=["sexual content"]),+ # Misinformation(types=["factual errors"]),+ # ExcessiveAgency(types=["functionality", "permissions"]),+ # UnauthorizedAccess(types=["sql injection", "bfla"]),+ # PIILeakage(types=["direct disclosure"]),+ # PromptLeakage(types=["instructions", "secrets and credentials"])]# Create Bad Likert Judge attack variants with different configurationsattacks = [# Test core vulnerability categories with different configurationsBadLikertJudge(category="illegal_activity", turns=3, enable_refinement=True),- BadLikertJudge(category="bias", turns=2, enable_refinement=False),- BadLikertJudge(category="toxicity", turns=3, enable_refinement=True),- BadLikertJudge(category="personal_safety", turns=2, enable_refinement=False),- BadLikertJudge(category="graphic_content", turns=3, enable_refinement=True),- BadLikertJudge(category="misinformation", turns=2, enable_refinement=False),- BadLikertJudge(category="excessive_agency", turns=3, enable_refinement=True),- BadLikertJudge(category="unauthorized_access", turns=2, enable_refinement=False),- BadLikertJudge(category="pii_leakage", turns=3, enable_refinement=True),- BadLikertJudge(category="prompt_leakage", turns=2, enable_refinement=False),+ # BadLikertJudge(category="bias", turns=2, enable_refinement=False),+ # BadLikertJudge(category="toxicity", turns=3, enable_refinement=True),+ # BadLikertJudge(category="personal_safety", turns=2, enable_refinement=False),+ # BadLikertJudge(category="graphic_content", turns=3, enable_refinement=True),+ # BadLikertJudge(category="misinformation", turns=2, enable_refinement=False),+ # BadLikertJudge(category="excessive_agency", turns=3, enable_refinement=True),+ # BadLikertJudge(category="unauthorized_access", turns=2, enable_refinement=False),+ # BadLikertJudge(category="pii_leakage", turns=3, enable_refinement=True),+ # BadLikertJudge(category="prompt_leakage", turns=2, enable_refinement=False),- # Test specific vulnerability subtypes- BadLikertJudge(category="weapons", turns=3, enable_refinement=True),- BadLikertJudge(category="religion", turns=2, enable_refinement=False),- BadLikertJudge(category="threats", turns=3, enable_refinement=True),- BadLikertJudge(category="self-harm", turns=2, enable_refinement=False),- BadLikertJudge(category="sql_injection", turns=3, enable_refinement=True),+ # # Test specific vulnerability subtypes+ # BadLikertJudge(category="weapons", turns=3, enable_refinement=True),+ # BadLikertJudge(category="religion", turns=2, enable_refinement=False),+ # BadLikertJudge(category="threats", turns=3, enable_refinement=True),+ # BadLikertJudge(category="self-harm", turns=2, enable_refinement=False),+ # BadLikertJudge(category="sql_injection", turns=3, enable_refinement=True),- # Test with malware/hacking categories (should map to unauthorized_access pattern)- BadLikertJudge(category="malware", turns=3, enable_refinement=True),- BadLikertJudge(category="hacking", turns=2, enable_refinement=False),+ # # Test with malware/hacking categories (should map to unauthorized_access pattern)+ # BadLikertJudge(category="malware", turns=3, enable_refinement=True),+ # BadLikertJudge(category="hacking", turns=2, enable_refinement=False),- # Test edge cases and custom categories- BadLikertJudge(category="violence", turns=3, enable_refinement=True),- BadLikertJudge(category="harassment", turns=2, enable_refinement=False),- BadLikertJudge(category="custom_harmful_category", turns=2, enable_refinement=False),+ # # Test edge cases and custom categories+ # BadLikertJudge(category="violence", turns=3, enable_refinement=True),+ # BadLikertJudge(category="harassment", turns=2, enable_refinement=False),+ # BadLikertJudge(category="custom_harmful_category", turns=2, enable_refinement=False),]print(f"📋 Created {len(attacks)} Bad Likert Judge attack variants:")@@ -128,10 +128,6 @@ async def main():max_concurrent=2 # Conservative concurrency)- # Temporarily monkey-patch the print method to avoid the error- original_print_method = red_teamer._print_risk_assessment- red_teamer._print_risk_assessment = lambda: print("📋 Risk assessment completed (printing bypassed)")-try:# Run red team attack with all variants - TEST EACH ATTACK/VULNERABILITY COMBINATIONprint("🔥 Running Bad Likert Judge red team attacks...")@@ -339,6 +335,9 @@ async def main():print(" • Target model may have strong safety measures")print(" • Consider testing with different model configurations")+ # Print the actual risk assessment now that the tuple issue is fixed+ red_teamer._print_risk_assessment()+except Exception as e:print(f"❌ Error during red team execution: {str(e)}")import traceback@@ -350,10 +349,6 @@ async def main():print(f"\n⚠️ Partial results available: {len(all_test_cases)} test cases")except:pass-- finally:- # Restore original method- red_teamer._print_risk_assessment = original_print_methodif __name__ == "__main__":asyncio.run(main())\ No newline at end of filediff --git a/tests/test_cli_config.py b/tests/test_cli_config.pynew file mode 100644index 00000000..8f5be4b2--- /dev/null+++ b/tests/test_cli_config.py@@ -0,0 +1,19 @@+import os+import importlib.util+from pathlib import Path++spec = importlib.util.spec_from_file_location(+ "config", Path(__file__).resolve().parents[1] / "deepteam/cli/config.py"+)+config = importlib.util.module_from_spec(spec)+spec.loader.exec_module(config)+++def test_config_roundtrip(tmp_path, monkeypatch):+ monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "cfg")+ config.set_key("TEST_KEY", "1")+ assert config.get_key("TEST_KEY") == "1"+ config.apply_env()+ assert os.environ.get("TEST_KEY") == "1"+ config.remove_key("TEST_KEY")+ assert config.get_key("TEST_KEY") is None\ No newline at end of filediff --git a/tests/test_cli_yaml.py b/tests/test_cli_yaml.pynew file mode 100644index 00000000..ba377321--- /dev/null+++ b/tests/test_cli_yaml.py@@ -0,0 +1,14 @@+from deepteam.cli import main as cli++sample = {+ "target": {"purpose": "t"},+ "default_vulnerabilities": [{"name": "Bias", "types": ["race"]}],+ "attacks": [{"name": "Prompt Injection", "weight": 1}],+}+++def test_build_objects():+ vulns = [cli._build_vulnerability(v) for v in sample["default_vulnerabilities"]]+ attacks = [cli._build_attack(a) for a in sample["attacks"]]+ assert vulns[0].get_name() == "Bias"+ assert attacks[0].get_name() == "Prompt Injection"diff --git a/tests/test_model_callback.py b/tests/test_model_callback.pynew file mode 100644index 00000000..72b2a586--- /dev/null+++ b/tests/test_model_callback.py@@ -0,0 +1,14 @@+from deepteam.cli.model_callback import load_model+from deepeval.models import GPTModel, OllamaModel++++def test_load_openai_str(monkeypatch):+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")+ m = load_model("gpt-3.5-turbo")+ assert isinstance(m, GPTModel)+++def test_load_ollama_dict():+ m = load_model({"provider": "ollama", "model": "phi3"})+ assert isinstance(m, OllamaModel)\ No newline at end of filediff --git a/uv.lock b/uv.lockindex a4a698e5..c10ca291 100644--- a/uv.lock+++ b/uv.lock@@ -496,13 +496,14 @@ wheels = [[[package]]name = "deepteam"-version = "0.1.3"+version = "0.1.5"source = { editable = "." }dependencies = [{ name = "aiohttp" },{ name = "deepeval" },{ name = "grpcio" },{ name = "openai" },+ { name = "pyyaml" },{ name = "requests" },{ name = "tabulate" },{ name = "tqdm" },@@ -529,6 +530,7 @@ requires-dist = [{ name = "openai", specifier = ">=1.76.2" },{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0" },+ { name = "pyyaml", specifier = ">=6.0.2" },{ name = "requests", specifier = ">=2.31.0" },{ name = "tabulate", specifier = ">=0.9.0" },{ name = "tqdm", specifier = ">=4.66.1" },@@ -1793,6 +1795,59 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" },]+[[package]]+name = "pyyaml"+version = "6.0.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" },+ { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" },+ { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" },+ { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" },+ { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" },+ { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" },+ { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" },+ { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" },+ { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" },+ { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" },+ { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" },+ { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" },+ { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" },+ { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" },+ { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" },+ { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" },+ { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" },+ { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" },+ { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" },+ { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" },+ { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" },+ { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" },+ { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" },+ { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" },+ { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" },+ { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" },+ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" },+ { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" },+ { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" },+ { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" },+ { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" },+ { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" },+ { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" },+ { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" },+ { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" },+ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },+ { url = "https://files.pythonhosted.org/packages/65/d8/b7a1db13636d7fb7d4ff431593c510c8b8fca920ade06ca8ef20015493c5/PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", size = 184777, upload-time = "2024-08-06T20:33:25.896Z" },+ { url = "https://files.pythonhosted.org/packages/0a/02/6ec546cd45143fdf9840b2c6be8d875116a64076218b61d68e12548e5839/PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", size = 172318, upload-time = "2024-08-06T20:33:27.212Z" },+ { url = "https://files.pythonhosted.org/packages/0e/9a/8cc68be846c972bda34f6c2a93abb644fb2476f4dcc924d52175786932c9/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", size = 720891, upload-time = "2024-08-06T20:33:28.974Z" },+ { url = "https://files.pythonhosted.org/packages/e9/6c/6e1b7f40181bc4805e2e07f4abc10a88ce4648e7e95ff1abe4ae4014a9b2/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", size = 722614, upload-time = "2024-08-06T20:33:34.157Z" },+ { url = "https://files.pythonhosted.org/packages/3d/32/e7bd8535d22ea2874cef6a81021ba019474ace0d13a4819c2a4bce79bd6a/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", size = 737360, upload-time = "2024-08-06T20:33:35.84Z" },+ { url = "https://files.pythonhosted.org/packages/d7/12/7322c1e30b9be969670b672573d45479edef72c9a0deac3bb2868f5d7469/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", size = 699006, upload-time = "2024-08-06T20:33:37.501Z" },+ { url = "https://files.pythonhosted.org/packages/82/72/04fcad41ca56491995076630c3ec1e834be241664c0c09a64c9a2589b507/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", size = 723577, upload-time = "2024-08-06T20:33:39.389Z" },+ { url = "https://files.pythonhosted.org/packages/ed/5e/46168b1f2757f1fcd442bc3029cd8767d88a98c9c05770d8b420948743bb/PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", size = 144593, upload-time = "2024-08-06T20:33:46.63Z" },+ { url = "https://files.pythonhosted.org/packages/19/87/5124b1c1f2412bb95c59ec481eaf936cd32f0fe2a7b16b97b81c4c017a6a/PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", size = 162312, upload-time = "2024-08-06T20:33:49.073Z" },+]+[[package]]name = "readme-renderer"version = "44.0"