SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
slaying the threading dragons (#1460)
TheTechromancer2c39b7f06a04
diff --git a/sfscan.py b/sfscan.pyindex b7836617..fbc3cada 100644--- a/sfscan.py+++ b/sfscan.py@@ -16,6 +16,7 @@ import queueimport tracebackfrom time import sleepfrom copy import deepcopy+from contextlib import suppressfrom collections import OrderedDictimport dns.resolver@@ -524,8 +525,17 @@ class SpiderFootScanner():}modules_waiting = sorted(modules_waiting.items(), key=lambda x: x[-1], reverse=True)modules_running = [m.__name__ for m in self.__moduleInstances.values() if m.running]+ modules_errored = [m.__name__ for m in self.__moduleInstances.values() if m.errorState]queues_empty = [qsize == 0 for m, qsize in modules_waiting]+ for mod in self.__moduleInstances.values():+ if mod.errorState and mod.incomingEventQueue is not None:+ self.__sf.debug(f"Clearing and unsetting incomingEventQueue for errored module {mod.__name__}.")+ with suppress(Exception):+ while 1:+ mod.incomingEventQueue.get_nowait()+ mod.incomingEventQueue = None+if not modules_running and not queues_empty:self.__sf.debug("Clearing queues for stalled/aborted modules.")for mod in self.__moduleInstances.values():@@ -539,9 +549,11 @@ class SpiderFootScanner():events_queued = ", ".join([f"{mod}: {qsize:,}" for mod, qsize in modules_waiting[:5] if qsize > 0])if not events_queued:events_queued = 'None'- self.__sf.debug(f"Events queued: {events_queued}")+ self.__sf.debug(f"Events queued: {sum([m[-1] for m in modules_waiting]):,} ({events_queued})")if modules_running:- self.__sf.debug(f"Modules running: {', '.join(modules_running)}")+ self.__sf.debug(f"Modules running: {len(modules_running):,} ({', '.join(modules_running)})")+ if modules_errored:+ self.__sf.debug(f"Modules errored: {len(modules_errored):,} ({', '.join(modules_errored)})")if all(queues_empty) and not modules_running:return Truediff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.pyindex 5b243002..60621f86 100644--- a/spiderfoot/plugin.py+++ b/spiderfoot/plugin.py@@ -403,17 +403,17 @@ class SpiderFootPlugin():while not self.checkForStop():try:sfEvent = self.incomingEventQueue.get_nowait()- self._running = True- if sfEvent == 'FINISHED':- self.sf.debug(f"{self.__name__}.threadWorker() got \"FINISHED\" from incomingEventQueue.")- self.finish()- else:- self.sf.debug(f"{self.__name__}.threadWorker() got event, {sfEvent.eventType}, from incomingEventQueue.")- self.handleEvent(sfEvent)- self._running = Falseexcept queue.Empty:sleep(.3)continue+ self._running = True+ if sfEvent == 'FINISHED':+ self.sf.debug(f"{self.__name__}.threadWorker() got \"FINISHED\" from incomingEventQueue.")+ self.finish()+ else:+ self.sf.debug(f"{self.__name__}.threadWorker() got event, {sfEvent.eventType}, from incomingEventQueue.")+ self.handleEvent(sfEvent)+ self._running = Falseexcept KeyboardInterrupt:self.sf.debug(f"Interrupted module {self.__name__}.")self._stopScanning = True@@ -550,7 +550,7 @@ class SpiderFootPlugin():for i in iterable:if self.stop:break- while 1:+ while not self.stop:try:q.put_nowait(i)break