Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

make traceScanner concurrent for sibling spans

A-Vamshi84bbba433bd5
diff --git a/deepteam/trace_scanner/trace_scanner.py b/deepteam/trace_scanner/trace_scanner.py
index d45a4fe9..11e264a0 100644
--- a/deepteam/trace_scanner/trace_scanner.py
+++ b/deepteam/trace_scanner/trace_scanner.py
@@ -1,63 +1,83 @@
+import asyncio
import json
-from typing import Dict, List, Any
+from dataclasses import dataclass, field
+from typing import Dict, List, Any, Optional
from deepeval.tracing.types import AgentSpan, LlmSpan, RetrieverSpan, ToolSpan, Trace, BaseSpan
from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics.utils import initialize_model
from .schema import BatchFinding, BatchFindingsList, SpanNode
from deepteam.utils import SPANS_CONTEXT_LIMIT
from deepteam.attacks.attack_simulator.utils import generate, a_generate
+
+@dataclass
+class BatchContext:
+ """Subtree-local accumulator for spans pending LLM evaluation.
+
+ Each traversal subtree owns one; concurrent sibling traversals never
+ share one, which keeps batch composition and size accounting correct
+ across the `await` inside an async flush.
+ """
+ batch: List[SpanNode] = field(default_factory=list)
+ size: int = 0
+
+
class TraceScanner:
def __init__(
self,
model: DeepEvalBaseLLM,
- using_native_model: bool,
template: Any,
limit: int = SPANS_CONTEXT_LIMIT,
+ max_concurrent: int = 10,
):
- self.model = model
- self.using_native_model = using_native_model
+ self.model, self.using_native_model = initialize_model(model)
self.template = template
self.limit = limit
-
+ self.max_concurrent = max_concurrent
+
# Internal State
- self._current_batch: List[SpanNode] = []
- self._current_batch_size: int = 0
self._findings: List[BatchFinding] = []
self._findings_by_span: Dict[str, List[BatchFinding]] = {}
+ self._semaphore: Optional[asyncio.Semaphore] = None
def process_trace(self, trace: Trace) -> List[BatchFinding]:
self._reset_state()
-
+ ctx = BatchContext()
+
for span in trace.root_spans:
- self._traverse_post_order(span)
-
+ self._merge_context(ctx, self._traverse_post_order(span))
+
trace_node = self._extract_trace_root(trace)
- self._add_to_batch_and_check(trace_node)
-
- if self._current_batch:
- self._flush_batch()
-
+ self._add_to_batch_and_check(ctx, trace_node)
+
+ if ctx.batch:
+ self._flush_batch(ctx)
+
return self._findings
async def a_process_trace(self, trace: Trace) -> List[BatchFinding]:
self._reset_state()
-
- for span in trace.root_spans:
- await self._a_traverse_post_order(span)
-
+ self._semaphore = asyncio.Semaphore(self.max_concurrent)
+ ctx = BatchContext()
+
+ # Root spans are siblings -> independent -> scanned concurrently.
+ child_ctxs = await asyncio.gather(
+ *(self._a_traverse_post_order(span) for span in trace.root_spans)
+ )
+ for child_ctx in child_ctxs:
+ await self._a_merge_context(ctx, child_ctx)
+
trace_node = self._extract_trace_root(trace)
- await self._a_add_to_batch_and_check(trace_node)
-
- if self._current_batch:
- await self._a_flush_batch()
-
+ await self._a_add_to_batch_and_check(ctx, trace_node)
+
+ if ctx.batch:
+ await self._a_flush_batch(ctx)
+
return self._findings
def _reset_state(self):
- self._current_batch = []
- self._current_batch_size = 0
self._findings = []
self._findings_by_span = {}
@@ -65,82 +85,104 @@ class TraceScanner:
# SYNCHRONOUS TRAVERSAL
# ---------------------------------------------------------
- def _traverse_post_order(self, span: BaseSpan):
+ def _traverse_post_order(self, span: BaseSpan) -> BatchContext:
+ ctx = BatchContext()
+
for child in span.children:
- self._traverse_post_order(child)
-
+ self._merge_context(ctx, self._traverse_post_order(child))
+
child_findings = self._get_child_findings(span.children)
span_node = self._extract_span_with_findings(span, child_findings)
-
- self._add_to_batch_and_check(span_node)
- def _add_to_batch_and_check(self, node: SpanNode):
+ self._add_to_batch_and_check(ctx, span_node)
+ return ctx
+
+ def _add_to_batch_and_check(self, ctx: BatchContext, node: SpanNode):
# Dump to dict strictly excluding None to save tokens
node_dict = node.model_dump(exclude_none=True)
node_str = json.dumps(node_dict)
node_size = len(node_str)
-
- if self._current_batch_size + node_size > self.limit and self._current_batch:
- self._flush_batch()
- self._current_batch.append(node)
- self._current_batch_size += node_size
+ if ctx.size + node_size > self.limit and ctx.batch:
+ self._flush_batch(ctx)
+
+ ctx.batch.append(node)
+ ctx.size += node_size
- if self._current_batch_size >= self.limit:
- self._flush_batch()
+ if ctx.size >= self.limit:
+ self._flush_batch(ctx)
- def _flush_batch(self):
- if not self._current_batch:
+ def _merge_context(self, parent_ctx: BatchContext, child_ctx: BatchContext):
+ """Fold a completed subtree's leftover (unflushed) nodes into the parent."""
+ for node in child_ctx.batch:
+ self._add_to_batch_and_check(parent_ctx, node)
+
+ def _flush_batch(self, ctx: BatchContext):
+ if not ctx.batch:
return
-
- batch_list = [node.model_dump(exclude_none=True) for node in self._current_batch]
+
+ batch_list = [node.model_dump(exclude_none=True) for node in ctx.batch]
batch_string = json.dumps(batch_list, indent=2)
prompt = self.template.generate_trace_batch_evaluation(batch_data=batch_string)
res: BatchFindingsList = generate(prompt, BatchFindingsList, self.model)
self._store_findings(res.findings)
- self._current_batch = []
- self._current_batch_size = 0
+ ctx.batch = []
+ ctx.size = 0
# ---------------------------------------------------------
# ASYNCHRONOUS TRAVERSAL
# ---------------------------------------------------------
- async def _a_traverse_post_order(self, span: BaseSpan):
- for child in span.children:
- await self._a_traverse_post_order(child)
-
+ async def _a_traverse_post_order(self, span: BaseSpan) -> BatchContext:
+ ctx = BatchContext()
+
+ if span.children:
+ # Sibling subtrees are independent -> scan them concurrently.
+ child_ctxs = await asyncio.gather(
+ *(self._a_traverse_post_order(child) for child in span.children)
+ )
+ for child_ctx in child_ctxs:
+ await self._a_merge_context(ctx, child_ctx)
+
child_findings = self._get_child_findings(span.children)
span_node = self._extract_span_with_findings(span, child_findings)
-
- await self._a_add_to_batch_and_check(span_node)
- async def _a_add_to_batch_and_check(self, node: SpanNode):
+ await self._a_add_to_batch_and_check(ctx, span_node)
+ return ctx
+
+ async def _a_add_to_batch_and_check(self, ctx: BatchContext, node: SpanNode):
node_dict = node.model_dump(exclude_none=True)
node_str = json.dumps(node_dict)
node_size = len(node_str)
-
- if self._current_batch_size + node_size > self.limit and self._current_batch:
- await self._a_flush_batch()
-
- self._current_batch.append(node)
- self._current_batch_size += node_size
-
- if self._current_batch_size >= self.limit:
- await self._a_flush_batch()
-
- async def _a_flush_batch(self):
- if not self._current_batch:
+
+ if ctx.size + node_size > self.limit and ctx.batch:
+ await self._a_flush_batch(ctx)
+
+ ctx.batch.append(node)
+ ctx.size += node_size
+
+ if ctx.size >= self.limit:
+ await self._a_flush_batch(ctx)
+
+ async def _a_merge_context(self, parent_ctx: BatchContext, child_ctx: BatchContext):
+ for node in child_ctx.batch:
+ await self._a_add_to_batch_and_check(parent_ctx, node)
+
+ async def _a_flush_batch(self, ctx: BatchContext):
+ if not ctx.batch:
return
-
- batch_list = [node.model_dump(exclude_none=True) for node in self._current_batch]
+
+ batch_list = [node.model_dump(exclude_none=True) for node in ctx.batch]
batch_string = json.dumps(batch_list, indent=2)
prompt = self.template.generate_trace_batch_evaluation(batch_data=batch_string)
- res: BatchFindingsList = await a_generate(prompt, BatchFindingsList, self.model)
+ # Bound the number of in-flight LLM calls across concurrent subtrees.
+ async with self._semaphore:
+ res: BatchFindingsList = await a_generate(prompt, BatchFindingsList, self.model)
self._store_findings(res.findings)
- self._current_batch = []
- self._current_batch_size = 0
+ ctx.batch = []
+ ctx.size = 0
# ---------------------------------------------------------
# UTILITIES & EXTRACTION
@@ -183,29 +225,29 @@ class TraceScanner:
def _collapse_io(self, parent_input: Any, parent_output: Any, children: List[BaseSpan]) -> tuple[Any, Any]:
"""
- Compares parent I/O with children I/O. If an exact match is found,
+ Compares parent I/O with children I/O. If an exact match is found,
returns None for that field to avoid redundant LLM evaluation / attribution.
"""
collapsed_input = parent_input
collapsed_output = parent_output
-
+
for child in children:
if collapsed_input is not None and collapsed_input == child.input:
collapsed_input = None
if collapsed_output is not None and collapsed_output == child.output:
collapsed_output = None
-
+
# Early exit if both are already collapsed
if collapsed_input is None and collapsed_output is None:
break
-
+
return collapsed_input, collapsed_output
def _extract_span_with_findings(self, span: BaseSpan, child_findings: List[BatchFinding]) -> SpanNode:
"""Strips useless metadata, keeps I/O, tools, and attaches child findings."""
c_input, c_output = self._collapse_io(span.input, span.output, span.children)
-
+
extracted = SpanNode(
spanUuid=span.uuid,
parentUuid=span.parent_uuid,
@@ -221,21 +263,21 @@ class TraceScanner:
tools_called=span.tools_called,
child_findings=child_findings if child_findings else None
)
-
+
# 2. Map Subclass-Specific Critical Fields
if isinstance(span, LlmSpan):
extracted.model = span.model
-
+
elif isinstance(span, AgentSpan):
extracted.available_tools = span.available_tools
extracted.agent_handoffs = span.agent_handoffs
-
+
elif isinstance(span, RetrieverSpan):
extracted.embedder = span.embedder
-
+
elif isinstance(span, ToolSpan):
extracted.description = span.description
-
+
return extracted
def _extract_trace_root(self, trace: Trace) -> SpanNode:
@@ -243,7 +285,7 @@ class TraceScanner:
root_findings = self._get_child_findings(trace.root_spans)
c_input, c_output = self._collapse_io(trace.input, trace.output, trace.root_spans)
-
+
return SpanNode(
spanUuid=trace.uuid,
parentUuid=None,
@@ -257,4 +299,4 @@ class TraceScanner:
expected_output=getattr(trace, "expected_output", None),
tools_called=[tc.model_dump(exclude_none=True) for tc in trace.tools_called] if getattr(trace, "tools_called", None) else None,
child_findings=root_findings if root_findings else None
- )
\ No newline at end of file
+ )
diff --git a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
index dd4b43a3..a65e7b2c 100644
--- a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
+++ b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
@@ -311,7 +311,6 @@ class AgentIdentityAbuse(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=AgentIdentityAbuseTemplate,
)
@@ -335,7 +334,6 @@ class AgentIdentityAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=AgentIdentityAbuseTemplate,
)
diff --git a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
index 68b30446..88e8c237 100644
--- a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
+++ b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
@@ -316,7 +316,6 @@ class AutonomousAgentDrift(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=AutonomousAgentDriftTemplate,
)
@@ -340,7 +339,6 @@ class AutonomousAgentDrift(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=AutonomousAgentDriftTemplate,
)
diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.py
index 1bf76a6d..6c488090 100644
--- a/deepteam/vulnerabilities/bfla/bfla.py
+++ b/deepteam/vulnerabilities/bfla/bfla.py
@@ -304,7 +304,6 @@ class BFLA(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=BFLATemplate,
)
@@ -328,7 +327,6 @@ class BFLA(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=BFLATemplate,
)
diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.py
index d86aca4c..e432e3c4 100644
--- a/deepteam/vulnerabilities/bias/bias.py
+++ b/deepteam/vulnerabilities/bias/bias.py
@@ -307,7 +307,6 @@ class Bias(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=BiasTemplate,
)
@@ -331,7 +330,6 @@ class Bias(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=BiasTemplate,
)
diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.py
index d4bd9e9e..a53588cb 100644
--- a/deepteam/vulnerabilities/bola/bola.py
+++ b/deepteam/vulnerabilities/bola/bola.py
@@ -308,7 +308,6 @@ class BOLA(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=BOLATemplate,
)
@@ -332,7 +331,6 @@ class BOLA(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=BOLATemplate,
)
diff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.py
index 05267016..4de28b08 100644
--- a/deepteam/vulnerabilities/child_protection/child_protection.py
+++ b/deepteam/vulnerabilities/child_protection/child_protection.py
@@ -308,7 +308,6 @@ class ChildProtection(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ChildProtectionTemplate,
)
@@ -332,7 +331,6 @@ class ChildProtection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ChildProtectionTemplate,
)
diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.py
index 0ca6c837..6ca9203a 100644
--- a/deepteam/vulnerabilities/competition/competition.py
+++ b/deepteam/vulnerabilities/competition/competition.py
@@ -311,7 +311,6 @@ class Competition(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=CompetitionTemplate,
)
@@ -335,7 +334,6 @@ class Competition(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=CompetitionTemplate,
)
diff --git a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
index a54ec543..8ebaac90 100644
--- a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
+++ b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
@@ -308,7 +308,6 @@ class CrossContextRetrieval(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=CrossContextRetrievalTemplate,
)
@@ -332,7 +331,6 @@ class CrossContextRetrieval(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=CrossContextRetrievalTemplate,
)
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index 3e8d2acb..3b09e87d 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -334,7 +334,6 @@ class CustomVulnerability(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=self._get_trace_template(),
)
@@ -358,7 +357,6 @@ class CustomVulnerability(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=self._get_trace_template(),
)
diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.py
index dde82475..78208df2 100644
--- a/deepteam/vulnerabilities/debug_access/debug_access.py
+++ b/deepteam/vulnerabilities/debug_access/debug_access.py
@@ -307,7 +307,6 @@ class DebugAccess(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=DebugAccessTemplate,
)
@@ -331,7 +330,6 @@ class DebugAccess(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=DebugAccessTemplate,
)
diff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.py
index 256e20f1..520e11d7 100644
--- a/deepteam/vulnerabilities/ethics/ethics.py
+++ b/deepteam/vulnerabilities/ethics/ethics.py
@@ -308,7 +308,6 @@ class Ethics(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=EthicsTemplate,
)
@@ -332,7 +331,6 @@ class Ethics(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=EthicsTemplate,
)
diff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
index 0e5259f6..d6afb660 100644
--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
@@ -305,7 +305,6 @@ class ExcessiveAgency(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ExcessiveAgencyTemplate,
)
@@ -329,7 +328,6 @@ class ExcessiveAgency(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ExcessiveAgencyTemplate,
)
diff --git a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
index 95db7855..d528ad0e 100644
--- a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
+++ b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
@@ -308,7 +308,6 @@ class ExploitToolAgent(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ExploitToolAgentTemplate,
)
@@ -332,7 +331,6 @@ class ExploitToolAgent(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ExploitToolAgentTemplate,
)
diff --git a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
index 652fd8b6..f4dc79c6 100644
--- a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
+++ b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
@@ -310,7 +310,6 @@ class ExternalSystemAbuse(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ExternalSystemAbuseTemplate,
)
@@ -334,7 +333,6 @@ class ExternalSystemAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ExternalSystemAbuseTemplate,
)
diff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.py
index f8adb450..7b0c7171 100644
--- a/deepteam/vulnerabilities/fairness/fairness.py
+++ b/deepteam/vulnerabilities/fairness/fairness.py
@@ -308,7 +308,6 @@ class Fairness(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=FairnessTemplate,
)
@@ -332,7 +331,6 @@ class Fairness(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=FairnessTemplate,
)
diff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.py
index 12a66d13..d4faad59 100644
--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py
+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py
@@ -308,7 +308,6 @@ class GoalTheft(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=GoalTheftTemplate,
)
@@ -332,7 +331,6 @@ class GoalTheft(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=GoalTheftTemplate,
)
diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.py
index 7423ab3d..89f3739a 100644
--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py
+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py
@@ -308,7 +308,6 @@ class GraphicContent(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=GraphicContentTemplate,
)
@@ -332,7 +331,6 @@ class GraphicContent(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=GraphicContentTemplate,
)
diff --git a/deepteam/vulnerabilities/hallucination/hallucination.py b/deepteam/vulnerabilities/hallucination/hallucination.py
index 03b62fed..1acd47e0 100644
--- a/deepteam/vulnerabilities/hallucination/hallucination.py
+++ b/deepteam/vulnerabilities/hallucination/hallucination.py
@@ -303,7 +303,6 @@ class Hallucination(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=HallucinationTemplate,
)
@@ -327,7 +326,6 @@ class Hallucination(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=HallucinationTemplate,
)
diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
index 8f98fb5f..24ec105c 100644
--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
@@ -315,7 +315,6 @@ class IllegalActivity(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=IllegalActivityTemplate,
)
@@ -339,7 +338,6 @@ class IllegalActivity(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=IllegalActivityTemplate,
)
diff --git a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
index 7845f161..ae59ab45 100644
--- a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
+++ b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
@@ -314,7 +314,6 @@ class IndirectInstruction(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=IndirectInstructionTemplate,
)
@@ -338,7 +337,6 @@ class IndirectInstruction(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=IndirectInstructionTemplate,
)
diff --git a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
index dad2687b..c31ad11d 100644
--- a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
+++ b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
@@ -326,7 +326,6 @@ class InsecureInterAgentCommunication(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=InsecureInterAgentCommunicationTemplate,
)
@@ -350,7 +349,6 @@ class InsecureInterAgentCommunication(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=InsecureInterAgentCommunicationTemplate,
)
diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
index cda48989..5cf6db7e 100644
--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
@@ -313,7 +313,6 @@ class IntellectualProperty(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=IntellectualPropertyTemplate,
)
@@ -337,7 +336,6 @@ class IntellectualProperty(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=IntellectualPropertyTemplate,
)
diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.py
index 2b77546d..73b15de2 100644
--- a/deepteam/vulnerabilities/misinformation/misinformation.py
+++ b/deepteam/vulnerabilities/misinformation/misinformation.py
@@ -307,7 +307,6 @@ class Misinformation(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=MisinformationTemplate,
)
@@ -331,7 +330,6 @@ class Misinformation(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=MisinformationTemplate,
)
diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.py
index fbd1a433..0e9e24e8 100644
--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py
+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py
@@ -307,7 +307,6 @@ class PersonalSafety(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=PersonalSafetyTemplate,
)
@@ -331,7 +330,6 @@ class PersonalSafety(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=PersonalSafetyTemplate,
)
diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
index 5ce12abb..0e812f9f 100644
--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
@@ -314,7 +314,6 @@ class PIILeakage(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=PIILeakageTemplate,
)
@@ -338,7 +337,6 @@ class PIILeakage(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=PIILeakageTemplate,
)
diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
index 3c7b01a2..5529004c 100644
--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
@@ -311,7 +311,6 @@ class PromptLeakage(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=PromptLeakageTemplate,
)
@@ -335,7 +334,6 @@ class PromptLeakage(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=PromptLeakageTemplate,
)
diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.py
index 7e18b9ec..b2fed6e2 100644
--- a/deepteam/vulnerabilities/rbac/rbac.py
+++ b/deepteam/vulnerabilities/rbac/rbac.py
@@ -308,7 +308,6 @@ class RBAC(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=RBACTemplate,
)
@@ -332,7 +331,6 @@ class RBAC(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=RBACTemplate,
)
diff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
index 2a8b620e..0fdde74f 100644
--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
@@ -312,7 +312,6 @@ class RecursiveHijacking(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=RecursiveHijackingTemplate,
)
@@ -336,7 +335,6 @@ class RecursiveHijacking(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=RecursiveHijackingTemplate,
)
diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.py
index fe76aa36..41b6383b 100644
--- a/deepteam/vulnerabilities/robustness/robustness.py
+++ b/deepteam/vulnerabilities/robustness/robustness.py
@@ -308,7 +308,6 @@ class Robustness(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=RobustnessTemplate,
)
@@ -332,7 +331,6 @@ class Robustness(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=RobustnessTemplate,
)
diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.py
index 61afd4cd..bb9c0a8a 100644
--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py
+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py
@@ -304,7 +304,6 @@ class ShellInjection(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ShellInjectionTemplate,
)
@@ -328,7 +327,6 @@ class ShellInjection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ShellInjectionTemplate,
)
diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.py
index 63c05afb..c490e066 100644
--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py
+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py
@@ -304,7 +304,6 @@ class SQLInjection(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=SQLInjectionTemplate,
)
@@ -328,7 +327,6 @@ class SQLInjection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=SQLInjectionTemplate,
)
diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.py
index d9be2e52..a575c426 100644
--- a/deepteam/vulnerabilities/ssrf/ssrf.py
+++ b/deepteam/vulnerabilities/ssrf/ssrf.py
@@ -302,7 +302,6 @@ class SSRF(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=SSRFTemplate,
)
@@ -326,7 +325,6 @@ class SSRF(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=SSRFTemplate,
)
diff --git a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
index 0cb1df8d..3d328b4b 100644
--- a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
+++ b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
@@ -310,7 +310,6 @@ class SystemReconnaissance(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=SystemReconnaissanceTemplate,
)
@@ -334,7 +333,6 @@ class SystemReconnaissance(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=SystemReconnaissanceTemplate,
)
diff --git a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
index 4db30be4..5f2eeecb 100644
--- a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
+++ b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
@@ -314,7 +314,6 @@ class ToolMetadataPoisoning(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ToolMetadataPoisoningTemplate,
)
@@ -338,7 +337,6 @@ class ToolMetadataPoisoning(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ToolMetadataPoisoningTemplate,
)
diff --git a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
index b14af701..c3f60ff0 100644
--- a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
+++ b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
@@ -316,7 +316,6 @@ class ToolOrchestrationAbuse(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ToolOrchestrationAbuseTemplate,
)
@@ -340,7 +339,6 @@ class ToolOrchestrationAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ToolOrchestrationAbuseTemplate,
)
diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.py
index 662d1f3c..f9329732 100644
--- a/deepteam/vulnerabilities/toxicity/toxicity.py
+++ b/deepteam/vulnerabilities/toxicity/toxicity.py
@@ -303,7 +303,6 @@ class Toxicity(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ToxicityTemplate,
)
@@ -327,7 +326,6 @@ class Toxicity(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=ToxicityTemplate,
)
diff --git a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
index 4d7b2acd..5b9a169a 100644
--- a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
+++ b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
@@ -317,7 +317,6 @@ class UnexpectedCodeExecution(BaseVulnerability):
)
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=UnexpectedCodeExecutionTemplate,
)
@@ -341,7 +340,6 @@ class UnexpectedCodeExecution(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
- using_native_model=self.using_native_model,
template=UnexpectedCodeExecutionTemplate,
)