DeepTeam is a framework to red team LLMs and AI agents.
make traceScanner concurrent for sibling spans
A-Vamshi84bbba433bd5
diff --git a/deepteam/trace_scanner/trace_scanner.py b/deepteam/trace_scanner/trace_scanner.pyindex d45a4fe9..11e264a0 100644--- a/deepteam/trace_scanner/trace_scanner.py+++ b/deepteam/trace_scanner/trace_scanner.py@@ -1,63 +1,83 @@+import asyncioimport json-from typing import Dict, List, Any+from dataclasses import dataclass, field+from typing import Dict, List, Any, Optionalfrom deepeval.tracing.types import AgentSpan, LlmSpan, RetrieverSpan, ToolSpan, Trace, BaseSpanfrom deepeval.models import DeepEvalBaseLLM+from deepeval.metrics.utils import initialize_modelfrom .schema import BatchFinding, BatchFindingsList, SpanNodefrom deepteam.utils import SPANS_CONTEXT_LIMITfrom deepteam.attacks.attack_simulator.utils import generate, a_generate++@dataclass+class BatchContext:+ """Subtree-local accumulator for spans pending LLM evaluation.++ Each traversal subtree owns one; concurrent sibling traversals never+ share one, which keeps batch composition and size accounting correct+ across the `await` inside an async flush.+ """+ batch: List[SpanNode] = field(default_factory=list)+ size: int = 0++class TraceScanner:def __init__(self,model: DeepEvalBaseLLM,- using_native_model: bool,template: Any,limit: int = SPANS_CONTEXT_LIMIT,+ max_concurrent: int = 10,):- self.model = model- self.using_native_model = using_native_model+ self.model, self.using_native_model = initialize_model(model)self.template = templateself.limit = limit-+ self.max_concurrent = max_concurrent+# Internal State- self._current_batch: List[SpanNode] = []- self._current_batch_size: int = 0self._findings: List[BatchFinding] = []self._findings_by_span: Dict[str, List[BatchFinding]] = {}+ self._semaphore: Optional[asyncio.Semaphore] = Nonedef process_trace(self, trace: Trace) -> List[BatchFinding]:self._reset_state()-+ ctx = BatchContext()+for span in trace.root_spans:- self._traverse_post_order(span)-+ self._merge_context(ctx, self._traverse_post_order(span))+trace_node = self._extract_trace_root(trace)- self._add_to_batch_and_check(trace_node)-- if self._current_batch:- self._flush_batch()-+ self._add_to_batch_and_check(ctx, trace_node)++ if ctx.batch:+ self._flush_batch(ctx)+return self._findingsasync def a_process_trace(self, trace: Trace) -> List[BatchFinding]:self._reset_state()-- for span in trace.root_spans:- await self._a_traverse_post_order(span)-+ self._semaphore = asyncio.Semaphore(self.max_concurrent)+ ctx = BatchContext()++ # Root spans are siblings -> independent -> scanned concurrently.+ child_ctxs = await asyncio.gather(+ *(self._a_traverse_post_order(span) for span in trace.root_spans)+ )+ for child_ctx in child_ctxs:+ await self._a_merge_context(ctx, child_ctx)+trace_node = self._extract_trace_root(trace)- await self._a_add_to_batch_and_check(trace_node)-- if self._current_batch:- await self._a_flush_batch()-+ await self._a_add_to_batch_and_check(ctx, trace_node)++ if ctx.batch:+ await self._a_flush_batch(ctx)+return self._findingsdef _reset_state(self):- self._current_batch = []- self._current_batch_size = 0self._findings = []self._findings_by_span = {}@@ -65,82 +85,104 @@ class TraceScanner:# SYNCHRONOUS TRAVERSAL# ---------------------------------------------------------- def _traverse_post_order(self, span: BaseSpan):+ def _traverse_post_order(self, span: BaseSpan) -> BatchContext:+ ctx = BatchContext()+for child in span.children:- self._traverse_post_order(child)-+ self._merge_context(ctx, self._traverse_post_order(child))+child_findings = self._get_child_findings(span.children)span_node = self._extract_span_with_findings(span, child_findings)-- self._add_to_batch_and_check(span_node)- def _add_to_batch_and_check(self, node: SpanNode):+ self._add_to_batch_and_check(ctx, span_node)+ return ctx++ def _add_to_batch_and_check(self, ctx: BatchContext, node: SpanNode):# Dump to dict strictly excluding None to save tokensnode_dict = node.model_dump(exclude_none=True)node_str = json.dumps(node_dict)node_size = len(node_str)-- if self._current_batch_size + node_size > self.limit and self._current_batch:- self._flush_batch()- self._current_batch.append(node)- self._current_batch_size += node_size+ if ctx.size + node_size > self.limit and ctx.batch:+ self._flush_batch(ctx)++ ctx.batch.append(node)+ ctx.size += node_size- if self._current_batch_size >= self.limit:- self._flush_batch()+ if ctx.size >= self.limit:+ self._flush_batch(ctx)- def _flush_batch(self):- if not self._current_batch:+ def _merge_context(self, parent_ctx: BatchContext, child_ctx: BatchContext):+ """Fold a completed subtree's leftover (unflushed) nodes into the parent."""+ for node in child_ctx.batch:+ self._add_to_batch_and_check(parent_ctx, node)++ def _flush_batch(self, ctx: BatchContext):+ if not ctx.batch:return-- batch_list = [node.model_dump(exclude_none=True) for node in self._current_batch]++ batch_list = [node.model_dump(exclude_none=True) for node in ctx.batch]batch_string = json.dumps(batch_list, indent=2)prompt = self.template.generate_trace_batch_evaluation(batch_data=batch_string)res: BatchFindingsList = generate(prompt, BatchFindingsList, self.model)self._store_findings(res.findings)- self._current_batch = []- self._current_batch_size = 0+ ctx.batch = []+ ctx.size = 0# ---------------------------------------------------------# ASYNCHRONOUS TRAVERSAL# ---------------------------------------------------------- async def _a_traverse_post_order(self, span: BaseSpan):- for child in span.children:- await self._a_traverse_post_order(child)-+ async def _a_traverse_post_order(self, span: BaseSpan) -> BatchContext:+ ctx = BatchContext()++ if span.children:+ # Sibling subtrees are independent -> scan them concurrently.+ child_ctxs = await asyncio.gather(+ *(self._a_traverse_post_order(child) for child in span.children)+ )+ for child_ctx in child_ctxs:+ await self._a_merge_context(ctx, child_ctx)+child_findings = self._get_child_findings(span.children)span_node = self._extract_span_with_findings(span, child_findings)-- await self._a_add_to_batch_and_check(span_node)- async def _a_add_to_batch_and_check(self, node: SpanNode):+ await self._a_add_to_batch_and_check(ctx, span_node)+ return ctx++ async def _a_add_to_batch_and_check(self, ctx: BatchContext, node: SpanNode):node_dict = node.model_dump(exclude_none=True)node_str = json.dumps(node_dict)node_size = len(node_str)-- if self._current_batch_size + node_size > self.limit and self._current_batch:- await self._a_flush_batch()-- self._current_batch.append(node)- self._current_batch_size += node_size-- if self._current_batch_size >= self.limit:- await self._a_flush_batch()-- async def _a_flush_batch(self):- if not self._current_batch:++ if ctx.size + node_size > self.limit and ctx.batch:+ await self._a_flush_batch(ctx)++ ctx.batch.append(node)+ ctx.size += node_size++ if ctx.size >= self.limit:+ await self._a_flush_batch(ctx)++ async def _a_merge_context(self, parent_ctx: BatchContext, child_ctx: BatchContext):+ for node in child_ctx.batch:+ await self._a_add_to_batch_and_check(parent_ctx, node)++ async def _a_flush_batch(self, ctx: BatchContext):+ if not ctx.batch:return-- batch_list = [node.model_dump(exclude_none=True) for node in self._current_batch]++ batch_list = [node.model_dump(exclude_none=True) for node in ctx.batch]batch_string = json.dumps(batch_list, indent=2)prompt = self.template.generate_trace_batch_evaluation(batch_data=batch_string)- res: BatchFindingsList = await a_generate(prompt, BatchFindingsList, self.model)+ # Bound the number of in-flight LLM calls across concurrent subtrees.+ async with self._semaphore:+ res: BatchFindingsList = await a_generate(prompt, BatchFindingsList, self.model)self._store_findings(res.findings)- self._current_batch = []- self._current_batch_size = 0+ ctx.batch = []+ ctx.size = 0# ---------------------------------------------------------# UTILITIES & EXTRACTION@@ -183,29 +225,29 @@ class TraceScanner:def _collapse_io(self, parent_input: Any, parent_output: Any, children: List[BaseSpan]) -> tuple[Any, Any]:"""- Compares parent I/O with children I/O. If an exact match is found,+ Compares parent I/O with children I/O. If an exact match is found,returns None for that field to avoid redundant LLM evaluation / attribution."""collapsed_input = parent_inputcollapsed_output = parent_output-+for child in children:if collapsed_input is not None and collapsed_input == child.input:collapsed_input = Noneif collapsed_output is not None and collapsed_output == child.output:collapsed_output = None-+# Early exit if both are already collapsedif collapsed_input is None and collapsed_output is None:break-+return collapsed_input, collapsed_outputdef _extract_span_with_findings(self, span: BaseSpan, child_findings: List[BatchFinding]) -> SpanNode:"""Strips useless metadata, keeps I/O, tools, and attaches child findings."""c_input, c_output = self._collapse_io(span.input, span.output, span.children)-+extracted = SpanNode(spanUuid=span.uuid,parentUuid=span.parent_uuid,@@ -221,21 +263,21 @@ class TraceScanner:tools_called=span.tools_called,child_findings=child_findings if child_findings else None)-+# 2. Map Subclass-Specific Critical Fieldsif isinstance(span, LlmSpan):extracted.model = span.model-+elif isinstance(span, AgentSpan):extracted.available_tools = span.available_toolsextracted.agent_handoffs = span.agent_handoffs-+elif isinstance(span, RetrieverSpan):extracted.embedder = span.embedder-+elif isinstance(span, ToolSpan):extracted.description = span.description-+return extracteddef _extract_trace_root(self, trace: Trace) -> SpanNode:@@ -243,7 +285,7 @@ class TraceScanner:root_findings = self._get_child_findings(trace.root_spans)c_input, c_output = self._collapse_io(trace.input, trace.output, trace.root_spans)-+return SpanNode(spanUuid=trace.uuid,parentUuid=None,@@ -257,4 +299,4 @@ class TraceScanner:expected_output=getattr(trace, "expected_output", None),tools_called=[tc.model_dump(exclude_none=True) for tc in trace.tools_called] if getattr(trace, "tools_called", None) else None,child_findings=root_findings if root_findings else None- )\ No newline at end of file+ )diff --git a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.pyindex dd4b43a3..a65e7b2c 100644--- a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py+++ b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py@@ -311,7 +311,6 @@ class AgentIdentityAbuse(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=AgentIdentityAbuseTemplate,)@@ -335,7 +334,6 @@ class AgentIdentityAbuse(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=AgentIdentityAbuseTemplate,)diff --git a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.pyindex 68b30446..88e8c237 100644--- a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py+++ b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py@@ -316,7 +316,6 @@ class AutonomousAgentDrift(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=AutonomousAgentDriftTemplate,)@@ -340,7 +339,6 @@ class AutonomousAgentDrift(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=AutonomousAgentDriftTemplate,)diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex 1bf76a6d..6c488090 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -304,7 +304,6 @@ class BFLA(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=BFLATemplate,)@@ -328,7 +327,6 @@ class BFLA(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=BFLATemplate,)diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex d86aca4c..e432e3c4 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -307,7 +307,6 @@ class Bias(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=BiasTemplate,)@@ -331,7 +330,6 @@ class Bias(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=BiasTemplate,)diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex d4bd9e9e..a53588cb 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -308,7 +308,6 @@ class BOLA(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=BOLATemplate,)@@ -332,7 +331,6 @@ class BOLA(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=BOLATemplate,)diff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.pyindex 05267016..4de28b08 100644--- a/deepteam/vulnerabilities/child_protection/child_protection.py+++ b/deepteam/vulnerabilities/child_protection/child_protection.py@@ -308,7 +308,6 @@ class ChildProtection(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ChildProtectionTemplate,)@@ -332,7 +331,6 @@ class ChildProtection(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ChildProtectionTemplate,)diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex 0ca6c837..6ca9203a 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -311,7 +311,6 @@ class Competition(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=CompetitionTemplate,)@@ -335,7 +334,6 @@ class Competition(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=CompetitionTemplate,)diff --git a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.pyindex a54ec543..8ebaac90 100644--- a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py+++ b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py@@ -308,7 +308,6 @@ class CrossContextRetrieval(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=CrossContextRetrievalTemplate,)@@ -332,7 +331,6 @@ class CrossContextRetrieval(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=CrossContextRetrievalTemplate,)diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex 3e8d2acb..3b09e87d 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -334,7 +334,6 @@ class CustomVulnerability(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=self._get_trace_template(),)@@ -358,7 +357,6 @@ class CustomVulnerability(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=self._get_trace_template(),)diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex dde82475..78208df2 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -307,7 +307,6 @@ class DebugAccess(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=DebugAccessTemplate,)@@ -331,7 +330,6 @@ class DebugAccess(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=DebugAccessTemplate,)diff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.pyindex 256e20f1..520e11d7 100644--- a/deepteam/vulnerabilities/ethics/ethics.py+++ b/deepteam/vulnerabilities/ethics/ethics.py@@ -308,7 +308,6 @@ class Ethics(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=EthicsTemplate,)@@ -332,7 +331,6 @@ class Ethics(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=EthicsTemplate,)diff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.pyindex 0e5259f6..d6afb660 100644--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py@@ -305,7 +305,6 @@ class ExcessiveAgency(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ExcessiveAgencyTemplate,)@@ -329,7 +328,6 @@ class ExcessiveAgency(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ExcessiveAgencyTemplate,)diff --git a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.pyindex 95db7855..d528ad0e 100644--- a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py+++ b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py@@ -308,7 +308,6 @@ class ExploitToolAgent(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ExploitToolAgentTemplate,)@@ -332,7 +331,6 @@ class ExploitToolAgent(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ExploitToolAgentTemplate,)diff --git a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.pyindex 652fd8b6..f4dc79c6 100644--- a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py+++ b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py@@ -310,7 +310,6 @@ class ExternalSystemAbuse(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ExternalSystemAbuseTemplate,)@@ -334,7 +333,6 @@ class ExternalSystemAbuse(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ExternalSystemAbuseTemplate,)diff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.pyindex f8adb450..7b0c7171 100644--- a/deepteam/vulnerabilities/fairness/fairness.py+++ b/deepteam/vulnerabilities/fairness/fairness.py@@ -308,7 +308,6 @@ class Fairness(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=FairnessTemplate,)@@ -332,7 +331,6 @@ class Fairness(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=FairnessTemplate,)diff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.pyindex 12a66d13..d4faad59 100644--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py@@ -308,7 +308,6 @@ class GoalTheft(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=GoalTheftTemplate,)@@ -332,7 +331,6 @@ class GoalTheft(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=GoalTheftTemplate,)diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 7423ab3d..89f3739a 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -308,7 +308,6 @@ class GraphicContent(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=GraphicContentTemplate,)@@ -332,7 +331,6 @@ class GraphicContent(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=GraphicContentTemplate,)diff --git a/deepteam/vulnerabilities/hallucination/hallucination.py b/deepteam/vulnerabilities/hallucination/hallucination.pyindex 03b62fed..1acd47e0 100644--- a/deepteam/vulnerabilities/hallucination/hallucination.py+++ b/deepteam/vulnerabilities/hallucination/hallucination.py@@ -303,7 +303,6 @@ class Hallucination(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=HallucinationTemplate,)@@ -327,7 +326,6 @@ class Hallucination(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=HallucinationTemplate,)diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex 8f98fb5f..24ec105c 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -315,7 +315,6 @@ class IllegalActivity(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=IllegalActivityTemplate,)@@ -339,7 +338,6 @@ class IllegalActivity(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=IllegalActivityTemplate,)diff --git a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.pyindex 7845f161..ae59ab45 100644--- a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py+++ b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py@@ -314,7 +314,6 @@ class IndirectInstruction(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=IndirectInstructionTemplate,)@@ -338,7 +337,6 @@ class IndirectInstruction(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=IndirectInstructionTemplate,)diff --git a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.pyindex dad2687b..c31ad11d 100644--- a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py+++ b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py@@ -326,7 +326,6 @@ class InsecureInterAgentCommunication(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=InsecureInterAgentCommunicationTemplate,)@@ -350,7 +349,6 @@ class InsecureInterAgentCommunication(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=InsecureInterAgentCommunicationTemplate,)diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex cda48989..5cf6db7e 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -313,7 +313,6 @@ class IntellectualProperty(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=IntellectualPropertyTemplate,)@@ -337,7 +336,6 @@ class IntellectualProperty(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=IntellectualPropertyTemplate,)diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex 2b77546d..73b15de2 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -307,7 +307,6 @@ class Misinformation(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=MisinformationTemplate,)@@ -331,7 +330,6 @@ class Misinformation(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=MisinformationTemplate,)diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex fbd1a433..0e9e24e8 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -307,7 +307,6 @@ class PersonalSafety(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=PersonalSafetyTemplate,)@@ -331,7 +330,6 @@ class PersonalSafety(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=PersonalSafetyTemplate,)diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex 5ce12abb..0e812f9f 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -314,7 +314,6 @@ class PIILeakage(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=PIILeakageTemplate,)@@ -338,7 +337,6 @@ class PIILeakage(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=PIILeakageTemplate,)diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex 3c7b01a2..5529004c 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -311,7 +311,6 @@ class PromptLeakage(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=PromptLeakageTemplate,)@@ -335,7 +334,6 @@ class PromptLeakage(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=PromptLeakageTemplate,)diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex 7e18b9ec..b2fed6e2 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -308,7 +308,6 @@ class RBAC(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=RBACTemplate,)@@ -332,7 +331,6 @@ class RBAC(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=RBACTemplate,)diff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.pyindex 2a8b620e..0fdde74f 100644--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py@@ -312,7 +312,6 @@ class RecursiveHijacking(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=RecursiveHijackingTemplate,)@@ -336,7 +335,6 @@ class RecursiveHijacking(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=RecursiveHijackingTemplate,)diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.pyindex fe76aa36..41b6383b 100644--- a/deepteam/vulnerabilities/robustness/robustness.py+++ b/deepteam/vulnerabilities/robustness/robustness.py@@ -308,7 +308,6 @@ class Robustness(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=RobustnessTemplate,)@@ -332,7 +331,6 @@ class Robustness(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=RobustnessTemplate,)diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex 61afd4cd..bb9c0a8a 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -304,7 +304,6 @@ class ShellInjection(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ShellInjectionTemplate,)@@ -328,7 +327,6 @@ class ShellInjection(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ShellInjectionTemplate,)diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex 63c05afb..c490e066 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -304,7 +304,6 @@ class SQLInjection(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=SQLInjectionTemplate,)@@ -328,7 +327,6 @@ class SQLInjection(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=SQLInjectionTemplate,)diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex d9be2e52..a575c426 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -302,7 +302,6 @@ class SSRF(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=SSRFTemplate,)@@ -326,7 +325,6 @@ class SSRF(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=SSRFTemplate,)diff --git a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.pyindex 0cb1df8d..3d328b4b 100644--- a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py+++ b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py@@ -310,7 +310,6 @@ class SystemReconnaissance(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=SystemReconnaissanceTemplate,)@@ -334,7 +333,6 @@ class SystemReconnaissance(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=SystemReconnaissanceTemplate,)diff --git a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.pyindex 4db30be4..5f2eeecb 100644--- a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py+++ b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py@@ -314,7 +314,6 @@ class ToolMetadataPoisoning(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ToolMetadataPoisoningTemplate,)@@ -338,7 +337,6 @@ class ToolMetadataPoisoning(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ToolMetadataPoisoningTemplate,)diff --git a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.pyindex b14af701..c3f60ff0 100644--- a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py+++ b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py@@ -316,7 +316,6 @@ class ToolOrchestrationAbuse(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ToolOrchestrationAbuseTemplate,)@@ -340,7 +339,6 @@ class ToolOrchestrationAbuse(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ToolOrchestrationAbuseTemplate,)diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex 662d1f3c..f9329732 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -303,7 +303,6 @@ class Toxicity(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ToxicityTemplate,)@@ -327,7 +326,6 @@ class Toxicity(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=ToxicityTemplate,)diff --git a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.pyindex 4d7b2acd..5b9a169a 100644--- a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py+++ b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py@@ -317,7 +317,6 @@ class UnexpectedCodeExecution(BaseVulnerability):)trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=UnexpectedCodeExecutionTemplate,)@@ -341,7 +340,6 @@ class UnexpectedCodeExecution(BaseVulnerability):trace_scanner = TraceScanner(model=self.evaluation_model,- using_native_model=self.using_native_model,template=UnexpectedCodeExecutionTemplate,)