Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

slaying the threading dragons (#1460)

TheTechromancer2c39b7f06a04
diff --git a/sfscan.py b/sfscan.py
index b7836617..fbc3cada 100644
--- a/sfscan.py
+++ b/sfscan.py
@@ -16,6 +16,7 @@ import queue
import traceback
from time import sleep
from copy import deepcopy
+from contextlib import suppress
from collections import OrderedDict
import dns.resolver
@@ -524,8 +525,17 @@ class SpiderFootScanner():
}
modules_waiting = sorted(modules_waiting.items(), key=lambda x: x[-1], reverse=True)
modules_running = [m.__name__ for m in self.__moduleInstances.values() if m.running]
+ modules_errored = [m.__name__ for m in self.__moduleInstances.values() if m.errorState]
queues_empty = [qsize == 0 for m, qsize in modules_waiting]
+ for mod in self.__moduleInstances.values():
+ if mod.errorState and mod.incomingEventQueue is not None:
+ self.__sf.debug(f"Clearing and unsetting incomingEventQueue for errored module {mod.__name__}.")
+ with suppress(Exception):
+ while 1:
+ mod.incomingEventQueue.get_nowait()
+ mod.incomingEventQueue = None
+
if not modules_running and not queues_empty:
self.__sf.debug("Clearing queues for stalled/aborted modules.")
for mod in self.__moduleInstances.values():
@@ -539,9 +549,11 @@ class SpiderFootScanner():
events_queued = ", ".join([f"{mod}: {qsize:,}" for mod, qsize in modules_waiting[:5] if qsize > 0])
if not events_queued:
events_queued = 'None'
- self.__sf.debug(f"Events queued: {events_queued}")
+ self.__sf.debug(f"Events queued: {sum([m[-1] for m in modules_waiting]):,} ({events_queued})")
if modules_running:
- self.__sf.debug(f"Modules running: {', '.join(modules_running)}")
+ self.__sf.debug(f"Modules running: {len(modules_running):,} ({', '.join(modules_running)})")
+ if modules_errored:
+ self.__sf.debug(f"Modules errored: {len(modules_errored):,} ({', '.join(modules_errored)})")
if all(queues_empty) and not modules_running:
return True
diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.py
index 5b243002..60621f86 100644
--- a/spiderfoot/plugin.py
+++ b/spiderfoot/plugin.py
@@ -403,17 +403,17 @@ class SpiderFootPlugin():
while not self.checkForStop():
try:
sfEvent = self.incomingEventQueue.get_nowait()
- self._running = True
- if sfEvent == 'FINISHED':
- self.sf.debug(f"{self.__name__}.threadWorker() got \"FINISHED\" from incomingEventQueue.")
- self.finish()
- else:
- self.sf.debug(f"{self.__name__}.threadWorker() got event, {sfEvent.eventType}, from incomingEventQueue.")
- self.handleEvent(sfEvent)
- self._running = False
except queue.Empty:
sleep(.3)
continue
+ self._running = True
+ if sfEvent == 'FINISHED':
+ self.sf.debug(f"{self.__name__}.threadWorker() got \"FINISHED\" from incomingEventQueue.")
+ self.finish()
+ else:
+ self.sf.debug(f"{self.__name__}.threadWorker() got event, {sfEvent.eventType}, from incomingEventQueue.")
+ self.handleEvent(sfEvent)
+ self._running = False
except KeyboardInterrupt:
self.sf.debug(f"Interrupted module {self.__name__}.")
self._stopScanning = True
@@ -550,7 +550,7 @@ class SpiderFootPlugin():
for i in iterable:
if self.stop:
break
- while 1:
+ while not self.stop:
try:
q.put_nowait(i)
break