DeepTeam is a framework to red team LLMs and AI agents.
fix return type
A-Vamshif072ec92573d
diff --git a/deepteam/trace_scanner/trace_scanner.py b/deepteam/trace_scanner/trace_scanner.pyindex 9d5d10cc..d45a4fe9 100644--- a/deepteam/trace_scanner/trace_scanner.py+++ b/deepteam/trace_scanner/trace_scanner.py@@ -24,9 +24,10 @@ class TraceScanner:# Internal Stateself._current_batch: List[SpanNode] = []self._current_batch_size: int = 0- self._all_findings: Dict[str, List[BatchFinding]] = {}+ self._findings: List[BatchFinding] = []+ self._findings_by_span: Dict[str, List[BatchFinding]] = {}- def process_trace(self, trace: Trace) -> Dict[str, List[BatchFinding]]:+ def process_trace(self, trace: Trace) -> List[BatchFinding]:self._reset_state()for span in trace.root_spans:@@ -38,9 +39,9 @@ class TraceScanner:if self._current_batch:self._flush_batch()- return self._all_findings+ return self._findings- async def a_process_trace(self, trace: Trace) -> Dict[str, List[BatchFinding]]:+ async def a_process_trace(self, trace: Trace) -> List[BatchFinding]:self._reset_state()for span in trace.root_spans:@@ -52,12 +53,13 @@ class TraceScanner:if self._current_batch:await self._a_flush_batch()- return self._all_findings+ return self._findingsdef _reset_state(self):self._current_batch = []self._current_batch_size = 0- self._all_findings = {}+ self._findings = []+ self._findings_by_span = {}# ---------------------------------------------------------# SYNCHRONOUS TRAVERSAL@@ -146,24 +148,37 @@ class TraceScanner:def _store_findings(self, findings: List[BatchFinding]):for finding in findings:- if finding.spanUuid not in self._all_findings:- self._all_findings[finding.spanUuid] = []-+ if finding.spanUuid not in self._findings_by_span:+ self._findings_by_span[finding.spanUuid] = []+# Filter out any older findings for this exact vulnerability type# because the newer finding from a higher-level batch is the final authority.- self._all_findings[finding.spanUuid] = [- f for f in self._all_findings[finding.spanUuid]- if not (f.vulnerability == finding.vulnerability and- f.vulnerabilityType == finding.vulnerabilityType)+ self._findings_by_span[finding.spanUuid] = [+ f+ for f in self._findings_by_span[finding.spanUuid]+ if not (+ f.vulnerability == finding.vulnerability+ and f.vulnerabilityType == finding.vulnerabilityType+ )]-- self._all_findings[finding.spanUuid].append(finding)+ self._findings_by_span[finding.spanUuid].append(finding)++ self._findings = [+ f+ for f in self._findings+ if not (+ f.spanUuid == finding.spanUuid+ and f.vulnerability == finding.vulnerability+ and f.vulnerabilityType == finding.vulnerabilityType+ )+ ]+ self._findings.append(finding)def _get_child_findings(self, children: List[BaseSpan]) -> List[BatchFinding]:findings = []for child in children:- if child.uuid in self._all_findings:- findings.extend(self._all_findings[child.uuid])+ if child.uuid in self._findings_by_span:+ findings.extend(self._findings_by_span[child.uuid])return findingsdef _collapse_io(self, parent_input: Any, parent_output: Any, children: List[BaseSpan]) -> tuple[Any, Any]:diff --git a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.pyindex f6c6912b..dd4b43a3 100644--- a/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py+++ b/deepteam/vulnerabilities/agent_identity_abuse/agent_identity_abuse.py@@ -296,7 +296,7 @@ class AgentIdentityAbuse(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Agent Identity & Trust Abuse vulnerabilities using bottoms-up batching."""@@ -318,16 +318,14 @@ class AgentIdentityAbuse(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Agent Identity & Trust Abuse vulnerabilities."""@@ -344,9 +342,7 @@ class AgentIdentityAbuse(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.pyindex 6e63a81f..68b30446 100644--- a/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py+++ b/deepteam/vulnerabilities/autonomous_agent_drift/autonomous_agent_drift.py@@ -301,7 +301,7 @@ class AutonomousAgentDrift(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Autonomous Agent Drift vulnerabilities using bottoms-up batching."""@@ -323,16 +323,14 @@ class AutonomousAgentDrift(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Autonomous Agent Drift vulnerabilities."""@@ -349,9 +347,7 @@ class AutonomousAgentDrift(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/bfla/bfla.py b/deepteam/vulnerabilities/bfla/bfla.pyindex 0814b2dc..1bf76a6d 100644--- a/deepteam/vulnerabilities/bfla/bfla.py+++ b/deepteam/vulnerabilities/bfla/bfla.py@@ -289,7 +289,7 @@ class BFLA(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for BFLA vulnerabilities using bottoms-up batching."""@@ -311,16 +311,14 @@ class BFLA(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for BFLA vulnerabilities."""@@ -337,9 +335,7 @@ class BFLA(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/bias/bias.py b/deepteam/vulnerabilities/bias/bias.pyindex 7b39aeda..d86aca4c 100644--- a/deepteam/vulnerabilities/bias/bias.py+++ b/deepteam/vulnerabilities/bias/bias.py@@ -292,7 +292,7 @@ class Bias(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Bias vulnerabilities using bottoms-up batching."""@@ -314,16 +314,14 @@ class Bias(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Bias vulnerabilities."""@@ -340,9 +338,7 @@ class Bias(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/bola/bola.py b/deepteam/vulnerabilities/bola/bola.pyindex 8a121925..d4bd9e9e 100644--- a/deepteam/vulnerabilities/bola/bola.py+++ b/deepteam/vulnerabilities/bola/bola.py@@ -293,7 +293,7 @@ class BOLA(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for BOLA vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class BOLA(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for BOLA vulnerabilities."""@@ -341,9 +339,7 @@ class BOLA(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/child_protection/child_protection.py b/deepteam/vulnerabilities/child_protection/child_protection.pyindex 15c122c3..05267016 100644--- a/deepteam/vulnerabilities/child_protection/child_protection.py+++ b/deepteam/vulnerabilities/child_protection/child_protection.py@@ -293,7 +293,7 @@ class ChildProtection(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Child Protection vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class ChildProtection(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Child Protection vulnerabilities."""@@ -341,9 +339,7 @@ class ChildProtection(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/competition/competition.py b/deepteam/vulnerabilities/competition/competition.pyindex 2d15d75f..0ca6c837 100644--- a/deepteam/vulnerabilities/competition/competition.py+++ b/deepteam/vulnerabilities/competition/competition.py@@ -296,7 +296,7 @@ class Competition(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Competition vulnerabilities using bottoms-up batching."""@@ -318,16 +318,14 @@ class Competition(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Competition vulnerabilities."""@@ -344,9 +342,7 @@ class Competition(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.pyindex 39d5e98a..a54ec543 100644--- a/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py+++ b/deepteam/vulnerabilities/cross_context_retrieval/cross_context_retrieval.py@@ -293,7 +293,7 @@ class CrossContextRetrieval(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Cross-Context Retrieval vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class CrossContextRetrieval(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Cross-Context Retrieval vulnerabilities."""@@ -341,9 +339,7 @@ class CrossContextRetrieval(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/custom/custom.py b/deepteam/vulnerabilities/custom/custom.pyindex f5e3bbcb..3e8d2acb 100644--- a/deepteam/vulnerabilities/custom/custom.py+++ b/deepteam/vulnerabilities/custom/custom.py@@ -319,7 +319,7 @@ class CustomVulnerability(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for custom vulnerabilities using bottoms-up batching."""@@ -341,16 +341,14 @@ class CustomVulnerability(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for custom vulnerabilities."""@@ -367,9 +365,7 @@ class CustomVulnerability(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/debug_access/debug_access.py b/deepteam/vulnerabilities/debug_access/debug_access.pyindex 304d7c84..dde82475 100644--- a/deepteam/vulnerabilities/debug_access/debug_access.py+++ b/deepteam/vulnerabilities/debug_access/debug_access.py@@ -292,7 +292,7 @@ class DebugAccess(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Debug Access vulnerabilities using bottoms-up batching."""@@ -314,16 +314,14 @@ class DebugAccess(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Debug Access vulnerabilities."""@@ -340,9 +338,7 @@ class DebugAccess(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/ethics/ethics.py b/deepteam/vulnerabilities/ethics/ethics.pyindex fe7a4fdd..256e20f1 100644--- a/deepteam/vulnerabilities/ethics/ethics.py+++ b/deepteam/vulnerabilities/ethics/ethics.py@@ -293,7 +293,7 @@ class Ethics(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Ethics vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class Ethics(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Ethics vulnerabilities."""@@ -341,9 +339,7 @@ class Ethics(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py b/deepteam/vulnerabilities/excessive_agency/excessive_agency.pyindex 5639e691..0e5259f6 100644--- a/deepteam/vulnerabilities/excessive_agency/excessive_agency.py+++ b/deepteam/vulnerabilities/excessive_agency/excessive_agency.py@@ -290,7 +290,7 @@ class ExcessiveAgency(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Excessive Agency vulnerabilities using bottoms-up batching."""@@ -312,16 +312,14 @@ class ExcessiveAgency(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Excessive Agency vulnerabilities."""@@ -338,9 +336,7 @@ class ExcessiveAgency(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.pyindex 60de114e..95db7855 100644--- a/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py+++ b/deepteam/vulnerabilities/exploit_tool_agent/exploit_tool_agent.py@@ -293,7 +293,7 @@ class ExploitToolAgent(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Exploit Tool Agent vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class ExploitToolAgent(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Exploit Tool Agent vulnerabilities."""@@ -341,9 +339,7 @@ class ExploitToolAgent(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.pyindex 013d59a4..652fd8b6 100644--- a/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py+++ b/deepteam/vulnerabilities/external_system_abuse/external_system_abuse.py@@ -295,7 +295,7 @@ class ExternalSystemAbuse(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for External System Abuse vulnerabilities using bottoms-up batching."""@@ -317,16 +317,14 @@ class ExternalSystemAbuse(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for External System Abuse vulnerabilities."""@@ -343,9 +341,7 @@ class ExternalSystemAbuse(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/fairness/fairness.py b/deepteam/vulnerabilities/fairness/fairness.pyindex 058d74f1..f8adb450 100644--- a/deepteam/vulnerabilities/fairness/fairness.py+++ b/deepteam/vulnerabilities/fairness/fairness.py@@ -293,7 +293,7 @@ class Fairness(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Fairness vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class Fairness(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Fairness vulnerabilities."""@@ -341,9 +339,7 @@ class Fairness(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/goal_theft/goal_theft.py b/deepteam/vulnerabilities/goal_theft/goal_theft.pyindex fef1dc02..12a66d13 100644--- a/deepteam/vulnerabilities/goal_theft/goal_theft.py+++ b/deepteam/vulnerabilities/goal_theft/goal_theft.py@@ -293,7 +293,7 @@ class GoalTheft(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Goal Theft vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class GoalTheft(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Goal Theft vulnerabilities."""@@ -341,9 +339,7 @@ class GoalTheft(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/graphic_content/graphic_content.py b/deepteam/vulnerabilities/graphic_content/graphic_content.pyindex 5ee3283f..7423ab3d 100644--- a/deepteam/vulnerabilities/graphic_content/graphic_content.py+++ b/deepteam/vulnerabilities/graphic_content/graphic_content.py@@ -293,7 +293,7 @@ class GraphicContent(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Graphic Content vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class GraphicContent(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Graphic Content vulnerabilities."""@@ -341,9 +339,7 @@ class GraphicContent(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/hallucination/hallucination.py b/deepteam/vulnerabilities/hallucination/hallucination.pyindex 234f4f59..03b62fed 100644--- a/deepteam/vulnerabilities/hallucination/hallucination.py+++ b/deepteam/vulnerabilities/hallucination/hallucination.py@@ -288,7 +288,7 @@ class Hallucination(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Hallucination vulnerabilities using bottoms-up batching."""@@ -310,16 +310,14 @@ class Hallucination(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Hallucination vulnerabilities."""@@ -336,9 +334,7 @@ class Hallucination(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py b/deepteam/vulnerabilities/illegal_activity/illegal_activity.pyindex f55cf994..8f98fb5f 100644--- a/deepteam/vulnerabilities/illegal_activity/illegal_activity.py+++ b/deepteam/vulnerabilities/illegal_activity/illegal_activity.py@@ -300,7 +300,7 @@ class IllegalActivity(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Illegal Activity vulnerabilities using bottoms-up batching."""@@ -322,16 +322,14 @@ class IllegalActivity(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Illegal Activity vulnerabilities."""@@ -348,9 +346,7 @@ class IllegalActivity(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.pyindex 14e86be8..7845f161 100644--- a/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py+++ b/deepteam/vulnerabilities/indirect_instruction/indirect_instruction.py@@ -299,7 +299,7 @@ class IndirectInstruction(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Indirect Instruction vulnerabilities using bottoms-up batching."""@@ -321,16 +321,14 @@ class IndirectInstruction(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Indirect Instruction vulnerabilities."""@@ -347,9 +345,7 @@ class IndirectInstruction(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.pyindex 77f30935..dad2687b 100644--- a/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py+++ b/deepteam/vulnerabilities/insecure_inter_agent_communication/insecure_inter_agent_communication.py@@ -311,7 +311,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Inter-Agent Communication Compromise vulnerabilities using bottoms-up batching."""@@ -333,16 +333,14 @@ class InsecureInterAgentCommunication(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Inter-Agent Communication Compromise vulnerabilities."""@@ -359,9 +357,7 @@ class InsecureInterAgentCommunication(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py b/deepteam/vulnerabilities/intellectual_property/intellectual_property.pyindex b71feb24..cda48989 100644--- a/deepteam/vulnerabilities/intellectual_property/intellectual_property.py+++ b/deepteam/vulnerabilities/intellectual_property/intellectual_property.py@@ -298,7 +298,7 @@ class IntellectualProperty(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Intellectual Property vulnerabilities using bottoms-up batching."""@@ -320,16 +320,14 @@ class IntellectualProperty(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Intellectual Property vulnerabilities."""@@ -346,9 +344,7 @@ class IntellectualProperty(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/misinformation/misinformation.py b/deepteam/vulnerabilities/misinformation/misinformation.pyindex d698d312..2b77546d 100644--- a/deepteam/vulnerabilities/misinformation/misinformation.py+++ b/deepteam/vulnerabilities/misinformation/misinformation.py@@ -292,7 +292,7 @@ class Misinformation(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Misinformation vulnerabilities using bottoms-up batching."""@@ -314,16 +314,14 @@ class Misinformation(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Misinformation vulnerabilities."""@@ -340,9 +338,7 @@ class Misinformation(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/personal_safety/personal_safety.py b/deepteam/vulnerabilities/personal_safety/personal_safety.pyindex b888d31c..fbd1a433 100644--- a/deepteam/vulnerabilities/personal_safety/personal_safety.py+++ b/deepteam/vulnerabilities/personal_safety/personal_safety.py@@ -292,7 +292,7 @@ class PersonalSafety(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Personal Safety vulnerabilities using bottoms-up batching."""@@ -314,16 +314,14 @@ class PersonalSafety(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Personal Safety vulnerabilities."""@@ -340,9 +338,7 @@ class PersonalSafety(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py b/deepteam/vulnerabilities/pii_leakage/pii_leakage.pyindex 9fcdb142..5ce12abb 100644--- a/deepteam/vulnerabilities/pii_leakage/pii_leakage.py+++ b/deepteam/vulnerabilities/pii_leakage/pii_leakage.py@@ -299,7 +299,7 @@ class PIILeakage(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for PII Leakage vulnerabilities using bottoms-up batching."""@@ -321,16 +321,14 @@ class PIILeakage(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for PII Leakage vulnerabilities."""@@ -347,9 +345,7 @@ class PIILeakage(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.pyindex 2f7c91a6..3c7b01a2 100644--- a/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py+++ b/deepteam/vulnerabilities/prompt_leakage/prompt_leakage.py@@ -296,7 +296,7 @@ class PromptLeakage(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Prompt Leakage vulnerabilities using bottoms-up batching."""@@ -318,16 +318,14 @@ class PromptLeakage(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Prompt Leakage vulnerabilities."""@@ -344,9 +342,7 @@ class PromptLeakage(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/rbac/rbac.py b/deepteam/vulnerabilities/rbac/rbac.pyindex 0da8ef99..7e18b9ec 100644--- a/deepteam/vulnerabilities/rbac/rbac.py+++ b/deepteam/vulnerabilities/rbac/rbac.py@@ -293,7 +293,7 @@ class RBAC(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for RBAC vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class RBAC(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for RBAC vulnerabilities."""@@ -341,9 +339,7 @@ class RBAC(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.pyindex c9a07c31..2a8b620e 100644--- a/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py+++ b/deepteam/vulnerabilities/recursive_hijacking/recursive_hijacking.py@@ -297,7 +297,7 @@ class RecursiveHijacking(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Recursive Hijacking vulnerabilities using bottoms-up batching."""@@ -319,16 +319,14 @@ class RecursiveHijacking(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Recursive Hijacking vulnerabilities."""@@ -345,9 +343,7 @@ class RecursiveHijacking(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/robustness/robustness.py b/deepteam/vulnerabilities/robustness/robustness.pyindex 4f15f84f..fe76aa36 100644--- a/deepteam/vulnerabilities/robustness/robustness.py+++ b/deepteam/vulnerabilities/robustness/robustness.py@@ -293,7 +293,7 @@ class Robustness(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Robustness vulnerabilities using bottoms-up batching."""@@ -315,16 +315,14 @@ class Robustness(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Robustness vulnerabilities."""@@ -341,9 +339,7 @@ class Robustness(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/shell_injection/shell_injection.py b/deepteam/vulnerabilities/shell_injection/shell_injection.pyindex d9a050c2..61afd4cd 100644--- a/deepteam/vulnerabilities/shell_injection/shell_injection.py+++ b/deepteam/vulnerabilities/shell_injection/shell_injection.py@@ -289,7 +289,7 @@ class ShellInjection(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Shell Injection vulnerabilities using bottoms-up batching."""@@ -311,16 +311,14 @@ class ShellInjection(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Shell Injection vulnerabilities."""@@ -337,9 +335,7 @@ class ShellInjection(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/sql_injection/sql_injection.py b/deepteam/vulnerabilities/sql_injection/sql_injection.pyindex eb50a52b..63c05afb 100644--- a/deepteam/vulnerabilities/sql_injection/sql_injection.py+++ b/deepteam/vulnerabilities/sql_injection/sql_injection.py@@ -289,7 +289,7 @@ class SQLInjection(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for SQL Injection vulnerabilities using bottoms-up batching."""@@ -311,16 +311,14 @@ class SQLInjection(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for SQL Injection vulnerabilities."""@@ -337,9 +335,7 @@ class SQLInjection(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/ssrf/ssrf.py b/deepteam/vulnerabilities/ssrf/ssrf.pyindex 50bca0b5..d9be2e52 100644--- a/deepteam/vulnerabilities/ssrf/ssrf.py+++ b/deepteam/vulnerabilities/ssrf/ssrf.py@@ -287,7 +287,7 @@ class SSRF(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for SSRF vulnerabilities using bottoms-up batching."""@@ -309,16 +309,14 @@ class SSRF(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for SSRF vulnerabilities."""@@ -335,9 +333,7 @@ class SSRF(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.pyindex 163ccde6..0cb1df8d 100644--- a/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py+++ b/deepteam/vulnerabilities/system_reconnaissance/system_reconnaissance.py@@ -295,7 +295,7 @@ class SystemReconnaissance(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for System Reconnaissance vulnerabilities using bottoms-up batching."""@@ -317,16 +317,14 @@ class SystemReconnaissance(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for System Reconnaissance vulnerabilities."""@@ -343,9 +341,7 @@ class SystemReconnaissance(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.pyindex 960fca8d..4db30be4 100644--- a/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py+++ b/deepteam/vulnerabilities/tool_metadata_poisoning/tool_metadata_poisoning.py@@ -299,7 +299,7 @@ class ToolMetadataPoisoning(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Tool Metadata Poisoning vulnerabilities using bottoms-up batching."""@@ -321,16 +321,14 @@ class ToolMetadataPoisoning(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Tool Metadata Poisoning vulnerabilities."""@@ -347,9 +345,7 @@ class ToolMetadataPoisoning(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.pyindex ce3621f6..b14af701 100644--- a/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py+++ b/deepteam/vulnerabilities/tool_orchestration_abuse/tool_orchestration_abuse.py@@ -301,7 +301,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Tool Orchestration Abuse vulnerabilities using bottoms-up batching."""@@ -323,16 +323,14 @@ class ToolOrchestrationAbuse(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Tool Orchestration Abuse vulnerabilities."""@@ -349,9 +347,7 @@ class ToolOrchestrationAbuse(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/toxicity/toxicity.py b/deepteam/vulnerabilities/toxicity/toxicity.pyindex 90a6cf38..662d1f3c 100644--- a/deepteam/vulnerabilities/toxicity/toxicity.py+++ b/deepteam/vulnerabilities/toxicity/toxicity.py@@ -288,7 +288,7 @@ class Toxicity(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Toxicity vulnerabilities using bottoms-up batching."""@@ -310,16 +310,14 @@ class Toxicity(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Toxicity vulnerabilities."""@@ -336,9 +334,7 @@ class Toxicity(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.pyindex 695346e7..4d7b2acd 100644--- a/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py+++ b/deepteam/vulnerabilities/unexpected_code_execution/unexpected_code_execution.py@@ -302,7 +302,7 @@ class UnexpectedCodeExecution(BaseVulnerability):def _assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Evaluates an entire execution trace for Unexpected Code Execution vulnerabilities using bottoms-up batching."""@@ -324,16 +324,14 @@ class UnexpectedCodeExecution(BaseVulnerability):findings = trace_scanner.process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsasync def _a_assess_trace(self,trace: Trace,- ) -> Dict[str, List[BatchFinding]]:+ ) -> List[BatchFinding]:"""Asynchronously evaluates an entire execution trace for Unexpected Code Execution vulnerabilities."""@@ -350,9 +348,7 @@ class UnexpectedCodeExecution(BaseVulnerability):findings = await trace_scanner.a_process_trace(trace)self.trace_findings = findings-- all_flat_findings = [f for batch in findings.values() for f in batch]- self.vulnerable = any(f.status == "unmitigated" for f in all_flat_findings)+ self.vulnerable = any(f.status == "unmitigated" for f in findings)return findingsdiff --git a/tests/test_core/test_batcher/test_batcher.py b/tests/test_core/test_batcher/test_batcher.pyindex 5dc5d2b7..352dffb1 100644--- a/tests/test_core/test_batcher/test_batcher.py+++ b/tests/test_core/test_batcher/test_batcher.py@@ -58,19 +58,15 @@ class TestTraceScanner:assert bias_vulnerability.vulnerable is Trueassert len(findings) >= 1-- all_findings = [f for batch in findings.values() for f in batch]- assert any(f.vulnerabilityType == "gender" and f.status == "unmitigated" for f in all_findings)+ assert any(f.vulnerabilityType == "gender" and f.status == "unmitigated" for f in findings)def test_guardrail_mitigation_no_root_breach(self, bias_vulnerability):trace = load_trace_fixture("guardrail_mitigated_trace")findings = bias_vulnerability._assess_trace(trace)-- all_findings = [f for batch in findings.values() for f in batch]- assert len(all_findings) > 0- assert all(f.status == "mitigated" for f in all_findings)+ assert len(findings) > 0+ assert all(f.status == "mitigated" for f in findings)assert bias_vulnerability.vulnerable is False@@ -80,7 +76,7 @@ class TestTraceScanner:findings = await bias_vulnerability._a_assess_trace(trace)- assert findings == {}+ assert findings == []assert bias_vulnerability.vulnerable is Falsedef test_io_collapsing_wrapper_edge_case(self, batch_evaluator_factory):