SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
EntryLevel MultiProcess
fallingcubesbc47147099a3
diff --git a/sf.py b/sf.pyindex c1fdac88..c279f8d6 100755--- a/sf.py+++ b/sf.py@@ -328,7 +328,7 @@ if __name__ == '__main__':t = SpiderFootScanner(target, target, targetType, scanId,modlist, cfg, dict())t.daemon = True- t.start()+ t.run()# If field headers weren't disabled, print themif not args.H and args.o != "json":diff --git a/sfscan.py b/sfscan.pyindex d93ce7dc..26fdff64 100644--- a/sfscan.py+++ b/sfscan.py@@ -22,7 +22,7 @@ from sflib import SpiderFoot, SpiderFootEvent, SpiderFootTarget, \SpiderFootPlugin, globalScanStatus# Eventually change this to be able to control multiple scan instances-class SpiderFootScanner(threading.Thread):+class SpiderFootScanner(): #threading.Thread):# Thread-safe storagets = None# Temporary storage@@ -33,8 +33,8 @@ class SpiderFootScanner(threading.Thread):globalOpts, moduleOpts):# Initialize the thread- threading.Thread.__init__(self, name="SF_" + scanId + \- str(random.SystemRandom().randint(100000, 999999)))+ #threading.Thread.__init__(self, name="SF_" + scanId + \+ # str(random.SystemRandom().randint(100000, 999999)))# Temporary data to be used in startScanself.temp = dict()@@ -44,115 +44,116 @@ class SpiderFootScanner(threading.Thread):self.temp['moduleList'] = moduleListself.temp['scanName'] = scanNameself.temp['scanId'] = scanId+ self.startScan()# Set the status of the currently running scan (if any)def setStatus(self, status, started=None, ended=None):- if self.ts is None:- print(("Internal Error: Status set attempted before " + \- "SpiderFootScanner was ready."))- exit(-1)-- self.ts.status = status- self.ts.dbh.scanInstanceSet(self.ts.scanId, started, ended, status)- globalScanStatus.setStatus(self.ts.scanId, status)+ #if self is None:+ # print(("Internal Error: Status set attempted before " + \+ # "SpiderFootScanner was ready."))+ # exit(-1)++ self.status = status+ self.dbh.scanInstanceSet(self.scanId, started, ended, status)+ #globalScanStatus.setStatus(self.scanId, status)return Nonedef run(self):self.startScan()def getId(self):- if hasattr(self.ts, 'scanId'):- return self.ts.scanId+ if hasattr(self, 'scanId'):+ return self.scanIdreturn None# Start running a scandef startScan(self):- global globalScanStatus-- self.ts = threading.local()- self.ts.moduleInstances = dict()- self.ts.sf = SpiderFoot(self.temp['config'])- self.ts.config = deepcopy(self.temp['config'])- self.ts.dbh = SpiderFootDb(self.temp['config'])- self.ts.targetValue = self.temp['targetValue']- self.ts.targetType = self.temp['targetType']- self.ts.moduleList = self.temp['moduleList']- self.ts.modconfig = dict()- self.ts.scanName = self.temp['scanName']- self.ts.scanId = self.temp['scanId']+ #global globalScanStatus++ #self = threading.local()+ self.moduleInstances = dict()+ self.sf = SpiderFoot(self.temp['config'])+ self.config = deepcopy(self.temp['config'])+ self.dbh = SpiderFootDb(self.temp['config'])+ self.targetValue = self.temp['targetValue']+ self.targetType = self.temp['targetType']+ self.moduleList = self.temp['moduleList']+ self.modconfig = dict()+ self.scanName = self.temp['scanName']+ self.scanId = self.temp['scanId']aborted = False- self.ts.sf.setDbh(self.ts.dbh)+ self.sf.setDbh(self.dbh)# Create a unique ID for this scan and create it in the back-end DB.- self.ts.sf.setGUID(self.ts.scanId)- self.ts.dbh.scanInstanceCreate(self.ts.scanId,- self.ts.scanName, self.ts.targetValue)+ self.sf.setGUID(self.scanId)+ self.dbh.scanInstanceCreate(self.scanId,+ self.scanName, self.targetValue)self.setStatus("STARTING", time.time() * 1000, None)# Create our target- target = SpiderFootTarget(self.ts.targetValue, self.ts.targetType)+ target = SpiderFootTarget(self.targetValue, self.targetType)# Save the config current set for this scan- self.ts.config['_modulesenabled'] = self.ts.moduleList- self.ts.dbh.scanConfigSet(self.ts.scanId,- self.ts.sf.configSerialize(deepcopy(self.ts.config)))+ self.config['_modulesenabled'] = self.moduleList+ self.dbh.scanConfigSet(self.scanId,+ self.sf.configSerialize(deepcopy(self.config)))- self.ts.sf.status("Scan [" + self.ts.scanId + "] initiated.")+ self.sf.status("Scan [" + self.scanId + "] initiated.")# moduleList = list of modules the user wants to runtry:# Process global options that point to other places for data# If a SOCKS server was specified, set it up- if self.ts.config['_socks1type'] != '':- socksDns = self.ts.config['_socks6dns']- socksAddr = self.ts.config['_socks2addr']- socksPort = int(self.ts.config['_socks3port'])- socksUsername = self.ts.config['_socks4user'] or ''- socksPassword = self.ts.config['_socks5pwd'] or ''+ if self.config['_socks1type'] != '':+ socksDns = self.config['_socks6dns']+ socksAddr = self.config['_socks2addr']+ socksPort = int(self.config['_socks3port'])+ socksUsername = self.config['_socks4user'] or ''+ socksPassword = self.config['_socks5pwd'] or ''creds = ""if socksUsername and socksPassword:creds = socksUsername + ":" + socksPassword + "@"proxy = creds + socksAddr + ":" + str(socksPort)- if self.ts.config['_socks1type'] == '4':+ if self.config['_socks1type'] == '4':proxy = 'socks4://' + proxy- elif self.ts.config['_socks1type'] == '5':+ elif self.config['_socks1type'] == '5':proxy = 'socks5://' + proxy- elif self.ts.config['_socks1type'] == 'HTTP':+ elif self.config['_socks1type'] == 'HTTP':proxy = 'http://' + proxy- elif self.ts.config['_socks1type'] == 'TOR':+ elif self.config['_socks1type'] == 'TOR':proxy = 'socks5h://' + proxy- self.ts.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \+ self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \"(" + socksUsername + ":" + socksPassword + ")")- self.ts.sf.updateSocket(proxy)+ self.sf.updateSocket(proxy)else:- self.ts.sf.revertSocket()+ self.sf.revertSocket()# Override the default DNS server- if self.ts.config['_dnsserver'] != "":+ if self.config['_dnsserver'] != "":res = dns.resolver.Resolver()- res.nameservers = [self.ts.config['_dnsserver']]+ res.nameservers = [self.config['_dnsserver']]dns.resolver.override_system_resolver(res)else:dns.resolver.restore_system_resolver()# Set the user agent- self.ts.config['_useragent'] = self.ts.sf.optValueToData(- self.ts.config['_useragent'])+ self.config['_useragent'] = self.sf.optValueToData(+ self.config['_useragent'])# Get internet TLDs- tlddata = self.ts.sf.cacheGet("internet_tlds",- self.ts.config['_internettlds_cache'])+ tlddata = self.sf.cacheGet("internet_tlds",+ self.config['_internettlds_cache'])# If it wasn't loadable from cache, load it from scratchif tlddata is None:- self.ts.config['_internettlds'] = self.ts.sf.optValueToData(- self.ts.config['_internettlds'])- self.ts.sf.cachePut("internet_tlds", self.ts.config['_internettlds'])+ self.config['_internettlds'] = self.sf.optValueToData(+ self.config['_internettlds'])+ self.sf.cachePut("internet_tlds", self.config['_internettlds'])else:- self.ts.config["_internettlds"] = tlddata.splitlines()+ self.config["_internettlds"] = tlddata.splitlines()- for modName in self.ts.moduleList:+ for modName in self.moduleList:if modName == '':continue@@ -160,51 +161,51 @@ class SpiderFootScanner(threading.Thread):module = __import__('modules.' + modName, globals(), locals(),[modName])except ImportError:- self.ts.sf.error("Failed to load module: " + modName, False)+ self.sf.error("Failed to load module: " + modName, False)continuemod = getattr(module, modName)()mod.__name__ = modName# Module may have been renamed or removed- if modName not in self.ts.config['__modules__']:+ if modName not in self.config['__modules__']:continue# Set up the module# Configuration is a combined global config with module-specific options- self.ts.modconfig[modName] = deepcopy(self.ts.config['__modules__'][modName]['opts'])- for opt in list(self.ts.config.keys()):- self.ts.modconfig[modName][opt] = deepcopy(self.ts.config[opt])+ self.modconfig[modName] = deepcopy(self.config['__modules__'][modName]['opts'])+ for opt in list(self.config.keys()):+ self.modconfig[modName][opt] = deepcopy(self.config[opt])mod.clearListeners() # clear any listener relationships from the past- mod.setup(self.ts.sf, self.ts.modconfig[modName])- mod.setDbh(self.ts.dbh)- mod.setScanId(self.ts.scanId)+ mod.setup(self.sf, self.modconfig[modName])+ mod.setDbh(self.dbh)+ mod.setScanId(self.scanId)# Give modules a chance to 'enrich' the original target with# aliases of that target.newTarget = mod.enrichTarget(target)if newTarget is not None:target = newTarget- self.ts.moduleInstances[modName] = mod+ self.moduleInstances[modName] = mod# Override the module's local socket module# to be the SOCKS one.- if self.ts.config['_socks1type'] != '':+ if self.config['_socks1type'] != '':mod._updateSocket(socket)# Set up event output filters if requested- if self.ts.config['__outputfilter']:- mod.setOutputFilter(self.ts.config['__outputfilter'])+ if self.config['__outputfilter']:+ mod.setOutputFilter(self.config['__outputfilter'])- self.ts.sf.status(modName + " module loaded.")+ self.sf.status(modName + " module loaded.")# Register listener modules and then start all modules sequentially- for module in list(self.ts.moduleInstances.values()):+ for module in list(self.moduleInstances.values()):# Register the target with the modulemodule.setTarget(target)- for listenerModule in list(self.ts.moduleInstances.values()):+ for listenerModule in list(self.moduleInstances.values()):# Careful not to register twice or you will get duplicate eventsif listenerModule in module._listenerModules:continue@@ -222,21 +223,21 @@ class SpiderFootScanner(threading.Thread):psMod.__name__ = "SpiderFoot UI"psMod.setTarget(target)psMod.clearListeners()- for mod in list(self.ts.moduleInstances.values()):+ for mod in list(self.moduleInstances.values()):if mod.watchedEvents() is not None:psMod.registerListener(mod)# Create the "ROOT" event which un-triggered modules will link events to- rootEvent = SpiderFootEvent("ROOT", self.ts.targetValue, "", None)+ rootEvent = SpiderFootEvent("ROOT", self.targetValue, "", None)psMod.notifyListeners(rootEvent)- firstEvent = SpiderFootEvent(self.ts.targetType, self.ts.targetValue,+ firstEvent = SpiderFootEvent(self.targetType, self.targetValue,"SpiderFoot UI", rootEvent)psMod.notifyListeners(firstEvent)# Special case.. check if an INTERNET_NAME is also a domain- if self.ts.targetType == 'INTERNET_NAME':- if self.ts.sf.isDomain(self.ts.targetValue, self.ts.config['_internettlds']):- firstEvent = SpiderFootEvent('DOMAIN_NAME', self.ts.targetValue,+ if self.targetType == 'INTERNET_NAME':+ if self.sf.isDomain(self.targetValue, self.config['_internettlds']):+ firstEvent = SpiderFootEvent('DOMAIN_NAME', self.targetValue,"SpiderFoot UI", rootEvent)psMod.notifyListeners(firstEvent)@@ -246,26 +247,24 @@ class SpiderFootScanner(threading.Thread):# Check in case the user requested to stop the scan between modules# initializing- for module in list(self.ts.moduleInstances.values()):+ for module in list(self.moduleInstances.values()):if module.checkForStop():self.setStatus('ABORTING')aborted = Truebreakif aborted:- self.ts.sf.status("Scan [" + self.ts.scanId + "] aborted.")+ self.sf.status("Scan [" + self.scanId + "] aborted.")self.setStatus("ABORTED", None, time.time() * 1000)else:- self.ts.sf.status("Scan [" + self.ts.scanId + "] completed.")+ self.sf.status("Scan [" + self.scanId + "] completed.")self.setStatus("FINISHED", None, time.time() * 1000)except BaseException as e:exc_type, exc_value, exc_traceback = sys.exc_info()- self.ts.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \+ self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \"encountered during scan. Please report this as a bug: " + \repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)- self.ts.sf.status("Scan [" + self.ts.scanId + "] failed: " + str(e))+ self.sf.status("Scan [" + self.scanId + "] failed: " + str(e))self.setStatus("ERROR-FAILED", None, time.time() * 1000)- self.ts.dbh.close()- del self.ts- del self.temp+ self.dbh.close()\ No newline at end of filediff --git a/sfwebui.py b/sfwebui.pyindex 4a7c7183..6a65de21 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -15,6 +15,7 @@ import htmlimport csvimport timeimport random+import multiprocessing as mpfrom secure import SecureHeadersfrom cherrypy import _cperrorfrom operator import itemgetter@@ -25,6 +26,7 @@ from sfdb import SpiderFootDbfrom sflib import SpiderFoot, globalScanStatusfrom sfscan import SpiderFootScannerfrom io import StringIO+mp.set_start_method("spawn", force=True)class SpiderFootWebUi:@@ -828,14 +830,14 @@ class SpiderFootWebUi:scantarget = scantarget.replace("\"", "")else:scantarget = scantarget.lower()- t = SpiderFootScanner(scanname, scantarget, targetType, scanId,- modlist, cfg, modopts)- t.start()+ p = mp.Process(target=SpiderFootScanner, args=(scanname, scantarget, targetType, scanId,+ modlist, cfg, modopts))+ p.start()# Wait until the scan has initialized- while globalScanStatus.getStatus(scanId) is None:- print("[info] Waiting for the scan to initialize...")- time.sleep(1)+ # while globalScanStatus.getStatus(scanId) is None:+ # print("[info] Waiting for the scan to initialize...")+ # time.sleep(1)if not cli:templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)