Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

update all vulnerabilities' assess_trace signature to accept previous detections

A-Vamshibb601041105c
diff --git a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
index 4cd86d8d..3a0bfc22 100644
--- a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
+++ b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
@@ -312,13 +312,14 @@ class AgentIdentityAbuse(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Agent Identity & Trust Abuse vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -326,6 +327,7 @@ class AgentIdentityAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=AgentIdentityAbuseTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -338,6 +340,7 @@ class AgentIdentityAbuse(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Agent Identity & Trust Abuse vulnerabilities.
@@ -349,6 +352,7 @@ class AgentIdentityAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=AgentIdentityAbuseTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
index 82a81595..f95e2da6 100644
--- a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
+++ b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
@@ -317,13 +317,14 @@ class AutonomousAgentDrift(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Autonomous Agent Drift vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -331,6 +332,7 @@ class AutonomousAgentDrift(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=AutonomousAgentDriftTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -343,6 +345,7 @@ class AutonomousAgentDrift(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Autonomous Agent Drift vulnerabilities.
@@ -354,6 +357,7 @@ class AutonomousAgentDrift(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=AutonomousAgentDriftTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/base_vulnerability.py b/deepteam/vulnerabilities/base_vulnerability.py
index dae48e16..59458d26 100644
--- a/deepteam/vulnerabilities/base_vulnerability.py
+++ b/deepteam/vulnerabilities/base_vulnerability.py
@@ -47,10 +47,10 @@ class BaseVulnerability(ABC):
async def a_simulate_attacks(self, purpose: str = None):
pass
- def _assess_trace(self, trace: Trace):
+ def _assess_trace(self, trace: Trace, previous_detections=None):
pass
- async def _a_assess_trace(self, trace: Trace):
+ async def _a_assess_trace(self, trace: Trace, previous_detections=None):
pass
def _get_metric(self) -> BaseRedTeamingMetric:
diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.py
index 9853a669..c9796c02 100644
--- a/deepteam/vulnerabilities/bfla/bfla.py
+++ b/deepteam/vulnerabilities/bfla/bfla.py
@@ -305,13 +305,14 @@ class BFLA(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for BFLA vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -319,6 +320,7 @@ class BFLA(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=BFLATemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -331,6 +333,7 @@ class BFLA(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for BFLA vulnerabilities.
@@ -342,6 +345,7 @@ class BFLA(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=BFLATemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.py
index 1d5c2e7a..5bd9c687 100644
--- a/deepteam/vulnerabilities/bias/bias.py
+++ b/deepteam/vulnerabilities/bias/bias.py
@@ -308,13 +308,14 @@ class Bias(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Bias vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -322,6 +323,7 @@ class Bias(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=BiasTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -334,6 +336,7 @@ class Bias(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Bias vulnerabilities.
@@ -345,6 +348,7 @@ class Bias(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=BiasTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.py
index a3aeaab2..f4e2c576 100644
--- a/deepteam/vulnerabilities/bola/bola.py
+++ b/deepteam/vulnerabilities/bola/bola.py
@@ -309,13 +309,14 @@ class BOLA(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for BOLA vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class BOLA(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=BOLATemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class BOLA(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for BOLA vulnerabilities.
@@ -346,6 +349,7 @@ class BOLA(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=BOLATemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.py
index 6da4623f..5b62dd97 100644
--- a/deepteam/vulnerabilities/child_protection/child_protection.py
+++ b/deepteam/vulnerabilities/child_protection/child_protection.py
@@ -309,13 +309,14 @@ class ChildProtection(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Child Protection vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class ChildProtection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ChildProtectionTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class ChildProtection(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Child Protection vulnerabilities.
@@ -346,6 +349,7 @@ class ChildProtection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ChildProtectionTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.py
index 19ba4dd0..96634fe2 100644
--- a/deepteam/vulnerabilities/competition/competition.py
+++ b/deepteam/vulnerabilities/competition/competition.py
@@ -312,13 +312,14 @@ class Competition(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Competition vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -326,6 +327,7 @@ class Competition(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=CompetitionTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -338,6 +340,7 @@ class Competition(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Competition vulnerabilities.
@@ -349,6 +352,7 @@ class Competition(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=CompetitionTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
index f42f18ea..d2917668 100644
--- a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
+++ b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
@@ -309,13 +309,14 @@ class CrossContextRetrieval(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Cross-Context Retrieval vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class CrossContextRetrieval(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=CrossContextRetrievalTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class CrossContextRetrieval(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Cross-Context Retrieval vulnerabilities.
@@ -346,6 +349,7 @@ class CrossContextRetrieval(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=CrossContextRetrievalTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index e7237396..1af3e1f7 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -337,13 +337,14 @@ class CustomVulnerability(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for custom vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -351,6 +352,7 @@ class CustomVulnerability(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=self._get_trace_template(),
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -363,6 +365,7 @@ class CustomVulnerability(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for custom vulnerabilities.
@@ -374,6 +377,7 @@ class CustomVulnerability(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=self._get_trace_template(),
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.py
index 431457e3..5c9f3462 100644
--- a/deepteam/vulnerabilities/debug_access/debug_access.py
+++ b/deepteam/vulnerabilities/debug_access/debug_access.py
@@ -308,13 +308,14 @@ class DebugAccess(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Debug Access vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -322,6 +323,7 @@ class DebugAccess(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=DebugAccessTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -334,6 +336,7 @@ class DebugAccess(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Debug Access vulnerabilities.
@@ -345,6 +348,7 @@ class DebugAccess(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=DebugAccessTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.py
index c5fee6f3..93e1ecb4 100644
--- a/deepteam/vulnerabilities/ethics/ethics.py
+++ b/deepteam/vulnerabilities/ethics/ethics.py
@@ -309,13 +309,14 @@ class Ethics(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Ethics vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class Ethics(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=EthicsTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class Ethics(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Ethics vulnerabilities.
@@ -346,6 +349,7 @@ class Ethics(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=EthicsTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
index d0e07901..693b252f 100644
--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
@@ -306,13 +306,14 @@ class ExcessiveAgency(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Excessive Agency vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -320,6 +321,7 @@ class ExcessiveAgency(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ExcessiveAgencyTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -332,6 +334,7 @@ class ExcessiveAgency(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Excessive Agency vulnerabilities.
@@ -343,6 +346,7 @@ class ExcessiveAgency(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ExcessiveAgencyTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
index c04d1fd3..12bcfab6 100644
--- a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
+++ b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
@@ -309,13 +309,14 @@ class ExploitToolAgent(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Exploit Tool Agent vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class ExploitToolAgent(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ExploitToolAgentTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class ExploitToolAgent(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Exploit Tool Agent vulnerabilities.
@@ -346,6 +349,7 @@ class ExploitToolAgent(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ExploitToolAgentTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
index 5ee0bfdf..eaac6ad1 100644
--- a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
+++ b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
@@ -311,13 +311,14 @@ class ExternalSystemAbuse(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for External System Abuse vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -325,6 +326,7 @@ class ExternalSystemAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ExternalSystemAbuseTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -337,6 +339,7 @@ class ExternalSystemAbuse(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for External System Abuse vulnerabilities.
@@ -348,6 +351,7 @@ class ExternalSystemAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ExternalSystemAbuseTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.py
index 31dcd6d3..07e62558 100644
--- a/deepteam/vulnerabilities/fairness/fairness.py
+++ b/deepteam/vulnerabilities/fairness/fairness.py
@@ -309,13 +309,14 @@ class Fairness(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Fairness vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class Fairness(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=FairnessTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class Fairness(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Fairness vulnerabilities.
@@ -346,6 +349,7 @@ class Fairness(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=FairnessTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.py
index 4466c5d6..337f318e 100644
--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py
+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py
@@ -309,13 +309,14 @@ class GoalTheft(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Goal Theft vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class GoalTheft(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=GoalTheftTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class GoalTheft(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Goal Theft vulnerabilities.
@@ -346,6 +349,7 @@ class GoalTheft(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=GoalTheftTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.py
index c7fc287f..233a4865 100644
--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py
+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py
@@ -309,13 +309,14 @@ class GraphicContent(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Graphic Content vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class GraphicContent(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=GraphicContentTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class GraphicContent(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Graphic Content vulnerabilities.
@@ -346,6 +349,7 @@ class GraphicContent(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=GraphicContentTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/hallucination/hallucination.py b/deepteam/vulnerabilities/hallucination/hallucination.py
index 8b0c44fa..b1c8f318 100644
--- a/deepteam/vulnerabilities/hallucination/hallucination.py
+++ b/deepteam/vulnerabilities/hallucination/hallucination.py
@@ -304,13 +304,14 @@ class Hallucination(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Hallucination vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -318,6 +319,7 @@ class Hallucination(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=HallucinationTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -330,6 +332,7 @@ class Hallucination(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Hallucination vulnerabilities.
@@ -341,6 +344,7 @@ class Hallucination(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=HallucinationTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
index 10bbbdd4..38a7f8b1 100644
--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
@@ -316,13 +316,14 @@ class IllegalActivity(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Illegal Activity vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -330,6 +331,7 @@ class IllegalActivity(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=IllegalActivityTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -342,6 +344,7 @@ class IllegalActivity(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Illegal Activity vulnerabilities.
@@ -353,6 +356,7 @@ class IllegalActivity(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=IllegalActivityTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
index f4c1a09a..454b010f 100644
--- a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
+++ b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
@@ -315,13 +315,14 @@ class IndirectInstruction(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Indirect Instruction vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -329,6 +330,7 @@ class IndirectInstruction(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=IndirectInstructionTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -341,6 +343,7 @@ class IndirectInstruction(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Indirect Instruction vulnerabilities.
@@ -352,6 +355,7 @@ class IndirectInstruction(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=IndirectInstructionTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
index 53fabb15..be621b76 100644
--- a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
+++ b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
@@ -327,13 +327,14 @@ class InsecureInterAgentCommunication(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Inter-Agent Communication Compromise vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -341,6 +342,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=InsecureInterAgentCommunicationTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -353,6 +355,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Inter-Agent Communication Compromise vulnerabilities.
@@ -364,6 +367,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=InsecureInterAgentCommunicationTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
index c0a9af19..398674bf 100644
--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
@@ -314,13 +314,14 @@ class IntellectualProperty(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Intellectual Property vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -328,6 +329,7 @@ class IntellectualProperty(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=IntellectualPropertyTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -340,6 +342,7 @@ class IntellectualProperty(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Intellectual Property vulnerabilities.
@@ -351,6 +354,7 @@ class IntellectualProperty(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=IntellectualPropertyTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.py
index 271dc75f..857f62f2 100644
--- a/deepteam/vulnerabilities/misinformation/misinformation.py
+++ b/deepteam/vulnerabilities/misinformation/misinformation.py
@@ -308,13 +308,14 @@ class Misinformation(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Misinformation vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -322,6 +323,7 @@ class Misinformation(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=MisinformationTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -334,6 +336,7 @@ class Misinformation(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Misinformation vulnerabilities.
@@ -345,6 +348,7 @@ class Misinformation(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=MisinformationTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.py
index 75911d56..4806813e 100644
--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py
+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py
@@ -308,13 +308,14 @@ class PersonalSafety(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Personal Safety vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -322,6 +323,7 @@ class PersonalSafety(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=PersonalSafetyTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -334,6 +336,7 @@ class PersonalSafety(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Personal Safety vulnerabilities.
@@ -345,6 +348,7 @@ class PersonalSafety(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=PersonalSafetyTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
index 5e88039b..54660134 100644
--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
@@ -315,13 +315,14 @@ class PIILeakage(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for PII Leakage vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -329,6 +330,7 @@ class PIILeakage(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=PIILeakageTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -341,6 +343,7 @@ class PIILeakage(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for PII Leakage vulnerabilities.
@@ -352,6 +355,7 @@ class PIILeakage(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=PIILeakageTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
index 150c77d6..c2e9448e 100644
--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
@@ -312,13 +312,14 @@ class PromptLeakage(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Prompt Leakage vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -326,6 +327,7 @@ class PromptLeakage(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=PromptLeakageTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -338,6 +340,7 @@ class PromptLeakage(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Prompt Leakage vulnerabilities.
@@ -349,6 +352,7 @@ class PromptLeakage(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=PromptLeakageTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.py
index 565d7a06..db6cce1b 100644
--- a/deepteam/vulnerabilities/rbac/rbac.py
+++ b/deepteam/vulnerabilities/rbac/rbac.py
@@ -309,13 +309,14 @@ class RBAC(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for RBAC vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class RBAC(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=RBACTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class RBAC(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for RBAC vulnerabilities.
@@ -346,6 +349,7 @@ class RBAC(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=RBACTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
index 4339b18b..0be49637 100644
--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
@@ -313,13 +313,14 @@ class RecursiveHijacking(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Recursive Hijacking vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -327,6 +328,7 @@ class RecursiveHijacking(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=RecursiveHijackingTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -339,6 +341,7 @@ class RecursiveHijacking(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Recursive Hijacking vulnerabilities.
@@ -350,6 +353,7 @@ class RecursiveHijacking(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=RecursiveHijackingTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.py
index e8481285..cf958e5e 100644
--- a/deepteam/vulnerabilities/robustness/robustness.py
+++ b/deepteam/vulnerabilities/robustness/robustness.py
@@ -309,13 +309,14 @@ class Robustness(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Robustness vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -323,6 +324,7 @@ class Robustness(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=RobustnessTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -335,6 +337,7 @@ class Robustness(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Robustness vulnerabilities.
@@ -346,6 +349,7 @@ class Robustness(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=RobustnessTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.py
index d5cc9d44..913bc949 100644
--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py
+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py
@@ -305,13 +305,14 @@ class ShellInjection(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Shell Injection vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -319,6 +320,7 @@ class ShellInjection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ShellInjectionTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -331,6 +333,7 @@ class ShellInjection(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Shell Injection vulnerabilities.
@@ -342,6 +345,7 @@ class ShellInjection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ShellInjectionTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.py
index 9b04e97b..fa32af39 100644
--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py
+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py
@@ -305,13 +305,14 @@ class SQLInjection(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for SQL Injection vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -319,6 +320,7 @@ class SQLInjection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=SQLInjectionTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -331,6 +333,7 @@ class SQLInjection(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for SQL Injection vulnerabilities.
@@ -342,6 +345,7 @@ class SQLInjection(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=SQLInjectionTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.py
index 72dd9501..e820683d 100644
--- a/deepteam/vulnerabilities/ssrf/ssrf.py
+++ b/deepteam/vulnerabilities/ssrf/ssrf.py
@@ -303,13 +303,14 @@ class SSRF(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for SSRF vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -317,6 +318,7 @@ class SSRF(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=SSRFTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -329,6 +331,7 @@ class SSRF(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for SSRF vulnerabilities.
@@ -340,6 +343,7 @@ class SSRF(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=SSRFTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
index 68f916b9..9a5e8bf7 100644
--- a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
+++ b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
@@ -311,13 +311,14 @@ class SystemReconnaissance(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for System Reconnaissance vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -325,6 +326,7 @@ class SystemReconnaissance(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=SystemReconnaissanceTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -337,6 +339,7 @@ class SystemReconnaissance(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for System Reconnaissance vulnerabilities.
@@ -348,6 +351,7 @@ class SystemReconnaissance(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=SystemReconnaissanceTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
index 8cf899ce..ee620d67 100644
--- a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
+++ b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
@@ -315,13 +315,14 @@ class ToolMetadataPoisoning(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Tool Metadata Poisoning vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -329,6 +330,7 @@ class ToolMetadataPoisoning(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ToolMetadataPoisoningTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -341,6 +343,7 @@ class ToolMetadataPoisoning(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Tool Metadata Poisoning vulnerabilities.
@@ -352,6 +355,7 @@ class ToolMetadataPoisoning(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ToolMetadataPoisoningTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
index 461a5619..c7afbed9 100644
--- a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
+++ b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
@@ -317,13 +317,14 @@ class ToolOrchestrationAbuse(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Tool Orchestration Abuse vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -331,6 +332,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ToolOrchestrationAbuseTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -343,6 +345,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Tool Orchestration Abuse vulnerabilities.
@@ -354,6 +357,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ToolOrchestrationAbuseTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.py
index 81aef3c1..10168410 100644
--- a/deepteam/vulnerabilities/toxicity/toxicity.py
+++ b/deepteam/vulnerabilities/toxicity/toxicity.py
@@ -304,13 +304,14 @@ class Toxicity(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Toxicity vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -318,6 +319,7 @@ class Toxicity(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ToxicityTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -330,6 +332,7 @@ class Toxicity(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Toxicity vulnerabilities.
@@ -341,6 +344,7 @@ class Toxicity(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=ToxicityTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)
diff --git a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
index 60a07636..90ef9d74 100644
--- a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
+++ b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
@@ -318,13 +318,14 @@ class UnexpectedCodeExecution(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Unexpected Code Execution vulnerabilities using bottoms-up batching.
"""
if self.async_mode:
loop = get_or_create_event_loop()
- return loop.run_until_complete(self._a_assess_trace(trace=trace))
+ return loop.run_until_complete(self._a_assess_trace(trace=trace, previous_detections=previous_detections))
self.evaluation_model, self.using_native_model = initialize_model(
self.evaluation_model
@@ -332,6 +333,7 @@ class UnexpectedCodeExecution(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=UnexpectedCodeExecutionTemplate,
+ previous_detections=previous_detections,
)
findings = trace_scanner.process_trace(trace)
@@ -344,6 +346,7 @@ class UnexpectedCodeExecution(BaseVulnerability):
async def _a_assess_trace(
self,
trace: Trace,
+ previous_detections: Optional[List[BatchFinding]] = None,
) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Unexpected Code Execution vulnerabilities.
@@ -355,6 +358,7 @@ class UnexpectedCodeExecution(BaseVulnerability):
trace_scanner = TraceScanner(
model=self.evaluation_model,
template=UnexpectedCodeExecutionTemplate,
+ previous_detections=previous_detections,
)
findings = await trace_scanner.a_process_trace(trace)