Skip to content

DeepTeam is a framework to red team LLMs and AI agents.

fix return type

A-Vamshif072ec92573d
diff --git a/deepteam/trace_scanner/trace_scanner.py b/deepteam/trace_scanner/trace_scanner.py
index 9d5d10cc..d45a4fe9 100644
--- a/deepteam/trace_scanner/trace_scanner.py
+++ b/deepteam/trace_scanner/trace_scanner.py
@@ -24,9 +24,10 @@ class TraceScanner:
# Internal State
self._current_batch: List[SpanNode] = []
self._current_batch_size: int = 0
- self._all_findings: Dict[str, List[BatchFinding]] = {}
+ self._findings: List[BatchFinding] = []
+ self._findings_by_span: Dict[str, List[BatchFinding]] = {}
- def process_trace(self, trace: Trace) -> Dict[str, List[BatchFinding]]:
+ def process_trace(self, trace: Trace) -> List[BatchFinding]:
self._reset_state()
for span in trace.root_spans:
@@ -38,9 +39,9 @@ class TraceScanner:
if self._current_batch:
self._flush_batch()
- return self._all_findings
+ return self._findings
- async def a_process_trace(self, trace: Trace) -> Dict[str, List[BatchFinding]]:
+ async def a_process_trace(self, trace: Trace) -> List[BatchFinding]:
self._reset_state()
for span in trace.root_spans:
@@ -52,12 +53,13 @@ class TraceScanner:
if self._current_batch:
await self._a_flush_batch()
- return self._all_findings
+ return self._findings
def _reset_state(self):
self._current_batch = []
self._current_batch_size = 0
- self._all_findings = {}
+ self._findings = []
+ self._findings_by_span = {}
# ---------------------------------------------------------
# SYNCHRONOUS TRAVERSAL
@@ -146,24 +148,37 @@ class TraceScanner:
def _store_findings(self, findings: List[BatchFinding]):
for finding in findings:
- if finding.spanUuid not in self._all_findings:
- self._all_findings[finding.spanUuid] = []
-
+ if finding.spanUuid not in self._findings_by_span:
+ self._findings_by_span[finding.spanUuid] = []
+
# Filter out any older findings for this exact vulnerability type
# because the newer finding from a higher-level batch is the final authority.
- self._all_findings[finding.spanUuid] = [
- f for f in self._all_findings[finding.spanUuid]
- if not (f.vulnerability == finding.vulnerability and
- f.vulnerabilityType == finding.vulnerabilityType)
+ self._findings_by_span[finding.spanUuid] = [
+ f
+ for f in self._findings_by_span[finding.spanUuid]
+ if not (
+ f.vulnerability == finding.vulnerability
+ and f.vulnerabilityType == finding.vulnerabilityType
+ )
]
-
- self._all_findings[finding.spanUuid].append(finding)
+ self._findings_by_span[finding.spanUuid].append(finding)
+
+ self._findings = [
+ f
+ for f in self._findings
+ if not (
+ f.spanUuid == finding.spanUuid
+ and f.vulnerability == finding.vulnerability
+ and f.vulnerabilityType == finding.vulnerabilityType
+ )
+ ]
+ self._findings.append(finding)
def _get_child_findings(self, children: List[BaseSpan]) -> List[BatchFinding]:
findings = []
for child in children:
- if child.uuid in self._all_findings:
- findings.extend(self._all_findings[child.uuid])
+ if child.uuid in self._findings_by_span:
+ findings.extend(self._findings_by_span[child.uuid])
return findings
def _collapse_io(self, parent_input: Any, parent_output: Any, children: List[BaseSpan]) -> tuple[Any, Any]:
diff --git a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
index f6c6912b..dd4b43a3 100644
--- a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
+++ b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py
@@ -296,7 +296,7 @@ class AgentIdentityAbuse(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Agent Identity & Trust Abuse vulnerabilities using bottoms-up batching.
"""
@@ -318,16 +318,14 @@ class AgentIdentityAbuse(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Agent Identity & Trust Abuse vulnerabilities.
"""
@@ -344,9 +342,7 @@ class AgentIdentityAbuse(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
index 6e63a81f..68b30446 100644
--- a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
+++ b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py
@@ -301,7 +301,7 @@ class AutonomousAgentDrift(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Autonomous Agent Drift vulnerabilities using bottoms-up batching.
"""
@@ -323,16 +323,14 @@ class AutonomousAgentDrift(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Autonomous Agent Drift vulnerabilities.
"""
@@ -349,9 +347,7 @@ class AutonomousAgentDrift(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.py
index 0814b2dc..1bf76a6d 100644
--- a/deepteam/vulnerabilities/bfla/bfla.py
+++ b/deepteam/vulnerabilities/bfla/bfla.py
@@ -289,7 +289,7 @@ class BFLA(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for BFLA vulnerabilities using bottoms-up batching.
"""
@@ -311,16 +311,14 @@ class BFLA(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for BFLA vulnerabilities.
"""
@@ -337,9 +335,7 @@ class BFLA(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.py
index 7b39aeda..d86aca4c 100644
--- a/deepteam/vulnerabilities/bias/bias.py
+++ b/deepteam/vulnerabilities/bias/bias.py
@@ -292,7 +292,7 @@ class Bias(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Bias vulnerabilities using bottoms-up batching.
"""
@@ -314,16 +314,14 @@ class Bias(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Bias vulnerabilities.
"""
@@ -340,9 +338,7 @@ class Bias(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.py
index 8a121925..d4bd9e9e 100644
--- a/deepteam/vulnerabilities/bola/bola.py
+++ b/deepteam/vulnerabilities/bola/bola.py
@@ -293,7 +293,7 @@ class BOLA(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for BOLA vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class BOLA(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for BOLA vulnerabilities.
"""
@@ -341,9 +339,7 @@ class BOLA(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.py
index 15c122c3..05267016 100644
--- a/deepteam/vulnerabilities/child_protection/child_protection.py
+++ b/deepteam/vulnerabilities/child_protection/child_protection.py
@@ -293,7 +293,7 @@ class ChildProtection(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Child Protection vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class ChildProtection(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Child Protection vulnerabilities.
"""
@@ -341,9 +339,7 @@ class ChildProtection(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.py
index 2d15d75f..0ca6c837 100644
--- a/deepteam/vulnerabilities/competition/competition.py
+++ b/deepteam/vulnerabilities/competition/competition.py
@@ -296,7 +296,7 @@ class Competition(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Competition vulnerabilities using bottoms-up batching.
"""
@@ -318,16 +318,14 @@ class Competition(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Competition vulnerabilities.
"""
@@ -344,9 +342,7 @@ class Competition(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
index 39d5e98a..a54ec543 100644
--- a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
+++ b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py
@@ -293,7 +293,7 @@ class CrossContextRetrieval(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Cross-Context Retrieval vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class CrossContextRetrieval(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Cross-Context Retrieval vulnerabilities.
"""
@@ -341,9 +339,7 @@ class CrossContextRetrieval(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.py
index f5e3bbcb..3e8d2acb 100644
--- a/deepteam/vulnerabilities/custom/custom.py
+++ b/deepteam/vulnerabilities/custom/custom.py
@@ -319,7 +319,7 @@ class CustomVulnerability(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for custom vulnerabilities using bottoms-up batching.
"""
@@ -341,16 +341,14 @@ class CustomVulnerability(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for custom vulnerabilities.
"""
@@ -367,9 +365,7 @@ class CustomVulnerability(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.py
index 304d7c84..dde82475 100644
--- a/deepteam/vulnerabilities/debug_access/debug_access.py
+++ b/deepteam/vulnerabilities/debug_access/debug_access.py
@@ -292,7 +292,7 @@ class DebugAccess(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Debug Access vulnerabilities using bottoms-up batching.
"""
@@ -314,16 +314,14 @@ class DebugAccess(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Debug Access vulnerabilities.
"""
@@ -340,9 +338,7 @@ class DebugAccess(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.py
index fe7a4fdd..256e20f1 100644
--- a/deepteam/vulnerabilities/ethics/ethics.py
+++ b/deepteam/vulnerabilities/ethics/ethics.py
@@ -293,7 +293,7 @@ class Ethics(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Ethics vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class Ethics(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Ethics vulnerabilities.
"""
@@ -341,9 +339,7 @@ class Ethics(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
index 5639e691..0e5259f6 100644
--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py
@@ -290,7 +290,7 @@ class ExcessiveAgency(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Excessive Agency vulnerabilities using bottoms-up batching.
"""
@@ -312,16 +312,14 @@ class ExcessiveAgency(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Excessive Agency vulnerabilities.
"""
@@ -338,9 +336,7 @@ class ExcessiveAgency(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
index 60de114e..95db7855 100644
--- a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
+++ b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py
@@ -293,7 +293,7 @@ class ExploitToolAgent(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Exploit Tool Agent vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class ExploitToolAgent(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Exploit Tool Agent vulnerabilities.
"""
@@ -341,9 +339,7 @@ class ExploitToolAgent(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
index 013d59a4..652fd8b6 100644
--- a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
+++ b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py
@@ -295,7 +295,7 @@ class ExternalSystemAbuse(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for External System Abuse vulnerabilities using bottoms-up batching.
"""
@@ -317,16 +317,14 @@ class ExternalSystemAbuse(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for External System Abuse vulnerabilities.
"""
@@ -343,9 +341,7 @@ class ExternalSystemAbuse(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.py
index 058d74f1..f8adb450 100644
--- a/deepteam/vulnerabilities/fairness/fairness.py
+++ b/deepteam/vulnerabilities/fairness/fairness.py
@@ -293,7 +293,7 @@ class Fairness(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Fairness vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class Fairness(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Fairness vulnerabilities.
"""
@@ -341,9 +339,7 @@ class Fairness(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.py
index fef1dc02..12a66d13 100644
--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py
+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py
@@ -293,7 +293,7 @@ class GoalTheft(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Goal Theft vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class GoalTheft(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Goal Theft vulnerabilities.
"""
@@ -341,9 +339,7 @@ class GoalTheft(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.py
index 5ee3283f..7423ab3d 100644
--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py
+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py
@@ -293,7 +293,7 @@ class GraphicContent(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Graphic Content vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class GraphicContent(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Graphic Content vulnerabilities.
"""
@@ -341,9 +339,7 @@ class GraphicContent(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/hallucination/hallucination.py b/deepteam/vulnerabilities/hallucination/hallucination.py
index 234f4f59..03b62fed 100644
--- a/deepteam/vulnerabilities/hallucination/hallucination.py
+++ b/deepteam/vulnerabilities/hallucination/hallucination.py
@@ -288,7 +288,7 @@ class Hallucination(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Hallucination vulnerabilities using bottoms-up batching.
"""
@@ -310,16 +310,14 @@ class Hallucination(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Hallucination vulnerabilities.
"""
@@ -336,9 +334,7 @@ class Hallucination(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
index f55cf994..8f98fb5f 100644
--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py
@@ -300,7 +300,7 @@ class IllegalActivity(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Illegal Activity vulnerabilities using bottoms-up batching.
"""
@@ -322,16 +322,14 @@ class IllegalActivity(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Illegal Activity vulnerabilities.
"""
@@ -348,9 +346,7 @@ class IllegalActivity(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
index 14e86be8..7845f161 100644
--- a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
+++ b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py
@@ -299,7 +299,7 @@ class IndirectInstruction(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Indirect Instruction vulnerabilities using bottoms-up batching.
"""
@@ -321,16 +321,14 @@ class IndirectInstruction(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Indirect Instruction vulnerabilities.
"""
@@ -347,9 +345,7 @@ class IndirectInstruction(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
index 77f30935..dad2687b 100644
--- a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
+++ b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py
@@ -311,7 +311,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Inter-Agent Communication Compromise vulnerabilities using bottoms-up batching.
"""
@@ -333,16 +333,14 @@ class InsecureInterAgentCommunication(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Inter-Agent Communication Compromise vulnerabilities.
"""
@@ -359,9 +357,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
index b71feb24..cda48989 100644
--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py
@@ -298,7 +298,7 @@ class IntellectualProperty(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Intellectual Property vulnerabilities using bottoms-up batching.
"""
@@ -320,16 +320,14 @@ class IntellectualProperty(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Intellectual Property vulnerabilities.
"""
@@ -346,9 +344,7 @@ class IntellectualProperty(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.py
index d698d312..2b77546d 100644
--- a/deepteam/vulnerabilities/misinformation/misinformation.py
+++ b/deepteam/vulnerabilities/misinformation/misinformation.py
@@ -292,7 +292,7 @@ class Misinformation(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Misinformation vulnerabilities using bottoms-up batching.
"""
@@ -314,16 +314,14 @@ class Misinformation(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Misinformation vulnerabilities.
"""
@@ -340,9 +338,7 @@ class Misinformation(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.py
index b888d31c..fbd1a433 100644
--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py
+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py
@@ -292,7 +292,7 @@ class PersonalSafety(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Personal Safety vulnerabilities using bottoms-up batching.
"""
@@ -314,16 +314,14 @@ class PersonalSafety(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Personal Safety vulnerabilities.
"""
@@ -340,9 +338,7 @@ class PersonalSafety(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
index 9fcdb142..5ce12abb 100644
--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py
@@ -299,7 +299,7 @@ class PIILeakage(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for PII Leakage vulnerabilities using bottoms-up batching.
"""
@@ -321,16 +321,14 @@ class PIILeakage(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for PII Leakage vulnerabilities.
"""
@@ -347,9 +345,7 @@ class PIILeakage(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
index 2f7c91a6..3c7b01a2 100644
--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py
@@ -296,7 +296,7 @@ class PromptLeakage(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Prompt Leakage vulnerabilities using bottoms-up batching.
"""
@@ -318,16 +318,14 @@ class PromptLeakage(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Prompt Leakage vulnerabilities.
"""
@@ -344,9 +342,7 @@ class PromptLeakage(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.py
index 0da8ef99..7e18b9ec 100644
--- a/deepteam/vulnerabilities/rbac/rbac.py
+++ b/deepteam/vulnerabilities/rbac/rbac.py
@@ -293,7 +293,7 @@ class RBAC(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for RBAC vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class RBAC(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for RBAC vulnerabilities.
"""
@@ -341,9 +339,7 @@ class RBAC(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
index c9a07c31..2a8b620e 100644
--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py
@@ -297,7 +297,7 @@ class RecursiveHijacking(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Recursive Hijacking vulnerabilities using bottoms-up batching.
"""
@@ -319,16 +319,14 @@ class RecursiveHijacking(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Recursive Hijacking vulnerabilities.
"""
@@ -345,9 +343,7 @@ class RecursiveHijacking(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.py
index 4f15f84f..fe76aa36 100644
--- a/deepteam/vulnerabilities/robustness/robustness.py
+++ b/deepteam/vulnerabilities/robustness/robustness.py
@@ -293,7 +293,7 @@ class Robustness(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Robustness vulnerabilities using bottoms-up batching.
"""
@@ -315,16 +315,14 @@ class Robustness(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Robustness vulnerabilities.
"""
@@ -341,9 +339,7 @@ class Robustness(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.py
index d9a050c2..61afd4cd 100644
--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py
+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py
@@ -289,7 +289,7 @@ class ShellInjection(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Shell Injection vulnerabilities using bottoms-up batching.
"""
@@ -311,16 +311,14 @@ class ShellInjection(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Shell Injection vulnerabilities.
"""
@@ -337,9 +335,7 @@ class ShellInjection(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.py
index eb50a52b..63c05afb 100644
--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py
+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py
@@ -289,7 +289,7 @@ class SQLInjection(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for SQL Injection vulnerabilities using bottoms-up batching.
"""
@@ -311,16 +311,14 @@ class SQLInjection(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for SQL Injection vulnerabilities.
"""
@@ -337,9 +335,7 @@ class SQLInjection(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.py
index 50bca0b5..d9be2e52 100644
--- a/deepteam/vulnerabilities/ssrf/ssrf.py
+++ b/deepteam/vulnerabilities/ssrf/ssrf.py
@@ -287,7 +287,7 @@ class SSRF(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for SSRF vulnerabilities using bottoms-up batching.
"""
@@ -309,16 +309,14 @@ class SSRF(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for SSRF vulnerabilities.
"""
@@ -335,9 +333,7 @@ class SSRF(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
index 163ccde6..0cb1df8d 100644
--- a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
+++ b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py
@@ -295,7 +295,7 @@ class SystemReconnaissance(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for System Reconnaissance vulnerabilities using bottoms-up batching.
"""
@@ -317,16 +317,14 @@ class SystemReconnaissance(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for System Reconnaissance vulnerabilities.
"""
@@ -343,9 +341,7 @@ class SystemReconnaissance(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
index 960fca8d..4db30be4 100644
--- a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
+++ b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py
@@ -299,7 +299,7 @@ class ToolMetadataPoisoning(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Tool Metadata Poisoning vulnerabilities using bottoms-up batching.
"""
@@ -321,16 +321,14 @@ class ToolMetadataPoisoning(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Tool Metadata Poisoning vulnerabilities.
"""
@@ -347,9 +345,7 @@ class ToolMetadataPoisoning(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
index ce3621f6..b14af701 100644
--- a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
+++ b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py
@@ -301,7 +301,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Tool Orchestration Abuse vulnerabilities using bottoms-up batching.
"""
@@ -323,16 +323,14 @@ class ToolOrchestrationAbuse(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Tool Orchestration Abuse vulnerabilities.
"""
@@ -349,9 +347,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.py
index 90a6cf38..662d1f3c 100644
--- a/deepteam/vulnerabilities/toxicity/toxicity.py
+++ b/deepteam/vulnerabilities/toxicity/toxicity.py
@@ -288,7 +288,7 @@ class Toxicity(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Toxicity vulnerabilities using bottoms-up batching.
"""
@@ -310,16 +310,14 @@ class Toxicity(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Toxicity vulnerabilities.
"""
@@ -336,9 +334,7 @@ class Toxicity(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
index 695346e7..4d7b2acd 100644
--- a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
+++ b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py
@@ -302,7 +302,7 @@ class UnexpectedCodeExecution(BaseVulnerability):
def _assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Evaluates an entire execution trace for Unexpected Code Execution vulnerabilities using bottoms-up batching.
"""
@@ -324,16 +324,14 @@ class UnexpectedCodeExecution(BaseVulnerability):
findings = trace_scanner.process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
async def _a_assess_trace(
self,
trace: Trace,
- ) -> Dict[str, List[BatchFinding]]:
+ ) -> List[BatchFinding]:
"""
Asynchronously evaluates an entire execution trace for Unexpected Code Execution vulnerabilities.
"""
@@ -350,9 +348,7 @@ class UnexpectedCodeExecution(BaseVulnerability):
findings = await trace_scanner.a_process_trace(trace)
self.trace_findings = findings
-
- all_flat_findings = [f for batch in findings.values() for f in batch]
- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)
+ self.vulnerable = any(f.status == "unmitigated" for f in findings)
return findings
diff --git a/tests/test_core/test_batcher/test_batcher.py b/tests/test_core/test_batcher/test_batcher.py
index 5dc5d2b7..352dffb1 100644
--- a/tests/test_core/test_batcher/test_batcher.py
+++ b/tests/test_core/test_batcher/test_batcher.py
@@ -58,19 +58,15 @@ class TestTraceScanner:
assert bias_vulnerability.vulnerable is True
assert len(findings) >= 1
-
- all_findings = [f for batch in findings.values() for f in batch]
- assert any(f.vulnerabilityType == "gender" and f.status == "unmitigated" for f in all_findings)
+ assert any(f.vulnerabilityType == "gender" and f.status == "unmitigated" for f in findings)
def test_guardrail_mitigation_no_root_breach(self, bias_vulnerability):
trace = load_trace_fixture("guardrail_mitigated_trace")
findings = bias_vulnerability._assess_trace(trace)
-
- all_findings = [f for batch in findings.values() for f in batch]
- assert len(all_findings) > 0
- assert all(f.status == "mitigated" for f in all_findings)
+ assert len(findings) > 0
+ assert all(f.status == "mitigated" for f in findings)
assert bias_vulnerability.vulnerable is False
@@ -80,7 +76,7 @@ class TestTraceScanner:
findings = await bias_vulnerability._a_assess_trace(trace)
- assert findings == {}
+ assert findings == []
assert bias_vulnerability.vulnerable is False
def test_io_collapsing_wrapper_edge_case(self, batch_evaluator_factory):