SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfscan: Cleanup __startScan error handling (#1364)
bcolesc4077d1e1899
diff --git a/sfscan.py b/sfscan.pyindex 4f865e62..f5d2ff22 100644--- a/sfscan.py+++ b/sfscan.py@@ -263,74 +263,109 @@ class SpiderFootScanner():aborted = Falseself.__setStatus("STARTING", time.time() * 1000, None)- self.__sf.status(f"Scan [{self.__scanId}] initiated.")+ self.__sf.status(f"Scan [{self.__scanId}] for '{self.__target.targetValue}' initiated.")if threaded:self.eventQueue = queue.Queue()- try:- # moduleList = list of modules the user wants to run- for modName in self.__moduleList:- if modName == '':- continue+ # moduleList = list of modules the user wants to run+ self.__sf.debug("Loading {len(self.__moduleList)} modules ...")+ for modName in self.__moduleList:+ if not modName:+ continue- try:- module = __import__('modules.' + modName, globals(), locals(), [modName])- except ImportError:- self.__sf.error(f"Failed to load module: {modName}")- continue+ # Module may have been renamed or removed+ if modName not in self.__config['__modules__']:+ self.__sf.error(f"Failed to load module: {modName}")+ continue+ try:+ module = __import__('modules.' + modName, globals(), locals(), [modName])+ except ImportError:+ self.__sf.error(f"Failed to load module: {modName}")+ continue++ try:mod = getattr(module, modName)()mod.__name__ = modName+ except Exception as e:+ self.__sf.error(f"Module {modName} initialization failed: {e}")+ continue- # Module may have been renamed or removed- if modName not in self.__config['__modules__']:- continue-- # Set up the module+ # Set up the module options, scan ID, database handle and listeners+ try:# Configuration is a combined global config with module-specific optionsself.__modconfig[modName] = deepcopy(self.__config['__modules__'][modName]['opts'])for opt in list(self.__config.keys()):self.__modconfig[modName][opt] = deepcopy(self.__config[opt])- mod.clearListeners() # clear any listener relationships from the past+ # clear any listener relationships from the past+ mod.clearListeners()mod.setup(self.__sf, self.__modconfig[modName])mod.setDbh(self.__dbh)mod.setScanId(self.__scanId)+ except Exception as e:+ self.__sf.error(f"Module {modName} initialization failed: {e}")+ continue- # Give modules a chance to 'enrich' the original target with- # aliases of that target.- newTarget = mod.enrichTarget(self.__target)- if newTarget is not None:- self.__target = newTarget- self.__moduleInstances[modName] = mod-- # Override the module's local socket module- # to be the SOCKS one.- if self.__config['_socks1type'] != '':+ # Override the module's local socket module to be the SOCKS one.+ if self.__config['_socks1type'] != '':+ try:mod._updateSocket(socket)+ except Exception as e:+ self.__sf.error(f"Module {modName} socket setup failed: {e}")+ continue- # Set up event output filters if requested- if self.__config['__outputfilter']:+ # Set up event output filters if requested+ if self.__config['__outputfilter']:+ try:mod.setOutputFilter(self.__config['__outputfilter'])+ except Exception as e:+ self.__sf.error(f"Module {modName} output filter setup failed: {e}")+ continue- # Register the target with the module+ # Give modules a chance to 'enrich' the original target with aliases of that target.+ try:+ newTarget = mod.enrichTarget(self.__target)+ if newTarget is not None:+ self.__target = newTarget+ except Exception as e:+ self.__sf.error(f"Module {modName} target enrichment failed: {e}")+ continue++ # Register the target with the module+ try:mod.setTarget(self.__target)+ except Exception as e:+ self.__sf.error(f"Module {modName} failed to set target '{self.__target}': {e}")+ continue- if threaded:- # Set up the outgoing event queue+ # Set up the outgoing event queue+ if threaded:+ try:mod.outgoingEventQueue = self.eventQueuemod.incomingEventQueue = queue.Queue()+ except Exception as e:+ self.__sf.error(f"Module {modName} event queue setup failed: {e}")+ continue++ self.__moduleInstances[modName] = mod+ self.__sf.status(f"{modName} module loaded.")++ self.__sf.debug(f"Scan [{self.__scanId}] loaded {len(self.__moduleInstances)} modules.")- self.__sf.status(modName + " module loaded.")+ if not self.__moduleInstances:+ self.__setStatus("ERROR-FAILED", None, time.time() * 1000)+ self.__dbh.close()+ return+ try:# sort modules by priorityself.__moduleInstances = OrderedDict(sorted(self.__moduleInstances.items(), key=lambda m: m[-1]._priority))if not threaded:# Register listener modules and then start all modules sequentiallyfor module in list(self.__moduleInstances.values()):-for listenerModule in list(self.__moduleInstances.values()):# Careful not to register twice or you will get duplicate eventsif listenerModule in module._listenerModules:diff --git a/test/unit/test_spiderfootscanner.py b/test/unit/test_spiderfootscanner.pyindex e3bbb233..bc43cf41 100644--- a/test/unit/test_spiderfootscanner.py+++ b/test/unit/test_spiderfootscanner.py@@ -21,19 +21,19 @@ class TestSpiderFootScanner(unittest.TestCase):scan_id = str(uuid.uuid4())module_list = ['sfp__stor_db']- sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)+ sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "INTERNET_NAME", module_list, opts, start=False)self.assertIsInstance(sfscan, SpiderFootScanner)self.assertEqual(sfscan.status, "INITIALIZING")- def test_init_argument_start_true_should_create_and_start_a_scan(self):+ def test_init_argument_start_true_with_no_valid_modules_should_set_scanstatus_to_failed(self):opts = self.default_optionsopts['__modules__'] = dict()scan_id = str(uuid.uuid4())- module_list = ['sfp__stor_db']+ module_list = ['invalid module']- sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=True)+ sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "INTERNET_NAME", module_list, opts, start=True)self.assertIsInstance(sfscan, SpiderFootScanner)- self.assertEqual(sfscan.status, "FINISHED")+ self.assertEqual(sfscan.status, "ERROR-FAILED")def test_init_argument_scanName_of_invalid_type_should_raise_TypeError(self):"""