Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

change model callback response handling to RTTurn

A-Vamshi1dbbea85b355
diff --git a/deepteam/red_teamer/red_teamer.py b/deepteam/red_teamer/red_teamer.py
index acc2956e..1e9ccad2 100644
--- a/deepteam/red_teamer/red_teamer.py
+++ b/deepteam/red_teamer/red_teamer.py
@@ -15,7 +15,7 @@ from deepeval.metrics.utils import initialize_model
from deepeval.dataset.golden import Golden
from deepteam.confident.api import Api, Endpoints
from deepteam.red_teamer.api import map_risk_assessment_to_api
-from deepteam.test_case import RTTestCase
+from deepteam.test_case import RTTestCase, RTTurn
from deepeval.utils import get_or_create_event_loop
from deepteam.frameworks.frameworks import AISafetyFramework
@@ -514,11 +514,14 @@ class RedTeamer:
try:
sig = inspect.signature(model_callback)
if "turns" in sig.parameters:
- actual_output = model_callback(
+ # We're calling model_callback here so we should expect the return object and parse fields here
+ model_response: RTTurn = model_callback(
simulated_test_case.input, simulated_test_case.turns
)
else:
- actual_output = model_callback(simulated_test_case.input)
+ model_response: RTTurn = model_callback(
+ simulated_test_case.input
+ )
except Exception:
if ignore_errors:
red_teaming_test_case.error = (
@@ -529,7 +532,11 @@ class RedTeamer:
raise
try:
- red_teaming_test_case.actual_output = actual_output
+ red_teaming_test_case.actual_output = model_response.content
+ red_teaming_test_case.retrieval_context = (
+ model_response.retrieval_context
+ )
+ red_teaming_test_case.tools_called = model_response.tools_called
metric.measure(red_teaming_test_case)
red_teaming_test_case.score = metric.score
red_teaming_test_case.reason = metric.reason
@@ -589,12 +596,14 @@ class RedTeamer:
try:
sig = inspect.signature(model_callback)
if "turns" in sig.parameters:
- actual_output = await model_callback(
- simulated_test_case.input, simulated_test_case.turns
+ model_response: RTTurn = (
+ await model_callback(
+ simulated_test_case.input, simulated_test_case.turns
+ )
)
else:
- actual_output = await model_callback(
- simulated_test_case.input
+ model_response: RTTurn = (
+ await model_callback(simulated_test_case.input)
)
except Exception:
if ignore_errors:
@@ -606,7 +615,11 @@ class RedTeamer:
raise
try:
- red_teaming_test_case.actual_output = actual_output
+ red_teaming_test_case.actual_output = model_response.content
+ red_teaming_test_case.retrieval_context = (
+ model_response.retrieval_context
+ )
+ red_teaming_test_case.tools_called = model_response.tools_called
await metric.a_measure(red_teaming_test_case)
red_teaming_test_case.score = metric.score
red_teaming_test_case.reason = metric.reason
diff --git a/deepteam/test_case/__init__.py b/deepteam/test_case/__init__.py
index a37d9a8d..275752ff 100644
--- a/deepteam/test_case/__init__.py
+++ b/deepteam/test_case/__init__.py
@@ -1,6 +1,7 @@
-from .test_case import RTTestCase, RTTurn
+from .test_case import RTTestCase, RTTurn, ToolCall
__all__ = [
"RTTestCase",
"RTTurn",
+ "ToolCall",
]
diff --git a/deepteam/test_case/test_case.py b/deepteam/test_case/test_case.py
index 5e111048..63b82dd2 100644
--- a/deepteam/test_case/test_case.py
+++ b/deepteam/test_case/test_case.py
@@ -1,6 +1,6 @@
from typing import List, Optional
from enum import Enum
-from deepeval.test_case import LLMTestCase, Turn
+from deepeval.test_case import LLMTestCase, Turn, ToolCall
class RTTurn(Turn):