Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

EntryLevel MultiProcess

fallingcubesbc47147099a3
diff --git a/sf.py b/sf.py
index c1fdac88..c279f8d6 100755
--- a/sf.py
+++ b/sf.py
@@ -328,7 +328,7 @@ if __name__ == '__main__':
t = SpiderFootScanner(target, target, targetType, scanId,
modlist, cfg, dict())
t.daemon = True
- t.start()
+ t.run()
# If field headers weren't disabled, print them
if not args.H and args.o != "json":
diff --git a/sfscan.py b/sfscan.py
index d93ce7dc..26fdff64 100644
--- a/sfscan.py
+++ b/sfscan.py
@@ -22,7 +22,7 @@ from sflib import SpiderFoot, SpiderFootEvent, SpiderFootTarget, \
SpiderFootPlugin, globalScanStatus
# Eventually change this to be able to control multiple scan instances
-class SpiderFootScanner(threading.Thread):
+class SpiderFootScanner(): #threading.Thread):
# Thread-safe storage
ts = None
# Temporary storage
@@ -33,8 +33,8 @@ class SpiderFootScanner(threading.Thread):
globalOpts, moduleOpts):
# Initialize the thread
- threading.Thread.__init__(self, name="SF_" + scanId + \
- str(random.SystemRandom().randint(100000, 999999)))
+ #threading.Thread.__init__(self, name="SF_" + scanId + \
+ # str(random.SystemRandom().randint(100000, 999999)))
# Temporary data to be used in startScan
self.temp = dict()
@@ -44,115 +44,116 @@ class SpiderFootScanner(threading.Thread):
self.temp['moduleList'] = moduleList
self.temp['scanName'] = scanName
self.temp['scanId'] = scanId
+ self.startScan()
# Set the status of the currently running scan (if any)
def setStatus(self, status, started=None, ended=None):
- if self.ts is None:
- print(("Internal Error: Status set attempted before " + \
- "SpiderFootScanner was ready."))
- exit(-1)
-
- self.ts.status = status
- self.ts.dbh.scanInstanceSet(self.ts.scanId, started, ended, status)
- globalScanStatus.setStatus(self.ts.scanId, status)
+ #if self is None:
+ # print(("Internal Error: Status set attempted before " + \
+ # "SpiderFootScanner was ready."))
+ # exit(-1)
+
+ self.status = status
+ self.dbh.scanInstanceSet(self.scanId, started, ended, status)
+ #globalScanStatus.setStatus(self.scanId, status)
return None
def run(self):
self.startScan()
def getId(self):
- if hasattr(self.ts, 'scanId'):
- return self.ts.scanId
+ if hasattr(self, 'scanId'):
+ return self.scanId
return None
# Start running a scan
def startScan(self):
- global globalScanStatus
-
- self.ts = threading.local()
- self.ts.moduleInstances = dict()
- self.ts.sf = SpiderFoot(self.temp['config'])
- self.ts.config = deepcopy(self.temp['config'])
- self.ts.dbh = SpiderFootDb(self.temp['config'])
- self.ts.targetValue = self.temp['targetValue']
- self.ts.targetType = self.temp['targetType']
- self.ts.moduleList = self.temp['moduleList']
- self.ts.modconfig = dict()
- self.ts.scanName = self.temp['scanName']
- self.ts.scanId = self.temp['scanId']
+ #global globalScanStatus
+
+ #self = threading.local()
+ self.moduleInstances = dict()
+ self.sf = SpiderFoot(self.temp['config'])
+ self.config = deepcopy(self.temp['config'])
+ self.dbh = SpiderFootDb(self.temp['config'])
+ self.targetValue = self.temp['targetValue']
+ self.targetType = self.temp['targetType']
+ self.moduleList = self.temp['moduleList']
+ self.modconfig = dict()
+ self.scanName = self.temp['scanName']
+ self.scanId = self.temp['scanId']
aborted = False
- self.ts.sf.setDbh(self.ts.dbh)
+ self.sf.setDbh(self.dbh)
# Create a unique ID for this scan and create it in the back-end DB.
- self.ts.sf.setGUID(self.ts.scanId)
- self.ts.dbh.scanInstanceCreate(self.ts.scanId,
- self.ts.scanName, self.ts.targetValue)
+ self.sf.setGUID(self.scanId)
+ self.dbh.scanInstanceCreate(self.scanId,
+ self.scanName, self.targetValue)
self.setStatus("STARTING", time.time() * 1000, None)
# Create our target
- target = SpiderFootTarget(self.ts.targetValue, self.ts.targetType)
+ target = SpiderFootTarget(self.targetValue, self.targetType)
# Save the config current set for this scan
- self.ts.config['_modulesenabled'] = self.ts.moduleList
- self.ts.dbh.scanConfigSet(self.ts.scanId,
- self.ts.sf.configSerialize(deepcopy(self.ts.config)))
+ self.config['_modulesenabled'] = self.moduleList
+ self.dbh.scanConfigSet(self.scanId,
+ self.sf.configSerialize(deepcopy(self.config)))
- self.ts.sf.status("Scan [" + self.ts.scanId + "] initiated.")
+ self.sf.status("Scan [" + self.scanId + "] initiated.")
# moduleList = list of modules the user wants to run
try:
# Process global options that point to other places for data
# If a SOCKS server was specified, set it up
- if self.ts.config['_socks1type'] != '':
- socksDns = self.ts.config['_socks6dns']
- socksAddr = self.ts.config['_socks2addr']
- socksPort = int(self.ts.config['_socks3port'])
- socksUsername = self.ts.config['_socks4user'] or ''
- socksPassword = self.ts.config['_socks5pwd'] or ''
+ if self.config['_socks1type'] != '':
+ socksDns = self.config['_socks6dns']
+ socksAddr = self.config['_socks2addr']
+ socksPort = int(self.config['_socks3port'])
+ socksUsername = self.config['_socks4user'] or ''
+ socksPassword = self.config['_socks5pwd'] or ''
creds = ""
if socksUsername and socksPassword:
creds = socksUsername + ":" + socksPassword + "@"
proxy = creds + socksAddr + ":" + str(socksPort)
- if self.ts.config['_socks1type'] == '4':
+ if self.config['_socks1type'] == '4':
proxy = 'socks4://' + proxy
- elif self.ts.config['_socks1type'] == '5':
+ elif self.config['_socks1type'] == '5':
proxy = 'socks5://' + proxy
- elif self.ts.config['_socks1type'] == 'HTTP':
+ elif self.config['_socks1type'] == 'HTTP':
proxy = 'http://' + proxy
- elif self.ts.config['_socks1type'] == 'TOR':
+ elif self.config['_socks1type'] == 'TOR':
proxy = 'socks5h://' + proxy
- self.ts.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \
+ self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \
"(" + socksUsername + ":" + socksPassword + ")")
- self.ts.sf.updateSocket(proxy)
+ self.sf.updateSocket(proxy)
else:
- self.ts.sf.revertSocket()
+ self.sf.revertSocket()
# Override the default DNS server
- if self.ts.config['_dnsserver'] != "":
+ if self.config['_dnsserver'] != "":
res = dns.resolver.Resolver()
- res.nameservers = [self.ts.config['_dnsserver']]
+ res.nameservers = [self.config['_dnsserver']]
dns.resolver.override_system_resolver(res)
else:
dns.resolver.restore_system_resolver()
# Set the user agent
- self.ts.config['_useragent'] = self.ts.sf.optValueToData(
- self.ts.config['_useragent'])
+ self.config['_useragent'] = self.sf.optValueToData(
+ self.config['_useragent'])
# Get internet TLDs
- tlddata = self.ts.sf.cacheGet("internet_tlds",
- self.ts.config['_internettlds_cache'])
+ tlddata = self.sf.cacheGet("internet_tlds",
+ self.config['_internettlds_cache'])
# If it wasn't loadable from cache, load it from scratch
if tlddata is None:
- self.ts.config['_internettlds'] = self.ts.sf.optValueToData(
- self.ts.config['_internettlds'])
- self.ts.sf.cachePut("internet_tlds", self.ts.config['_internettlds'])
+ self.config['_internettlds'] = self.sf.optValueToData(
+ self.config['_internettlds'])
+ self.sf.cachePut("internet_tlds", self.config['_internettlds'])
else:
- self.ts.config["_internettlds"] = tlddata.splitlines()
+ self.config["_internettlds"] = tlddata.splitlines()
- for modName in self.ts.moduleList:
+ for modName in self.moduleList:
if modName == '':
continue
@@ -160,51 +161,51 @@ class SpiderFootScanner(threading.Thread):
module = __import__('modules.' + modName, globals(), locals(),
[modName])
except ImportError:
- self.ts.sf.error("Failed to load module: " + modName, False)
+ self.sf.error("Failed to load module: " + modName, False)
continue
mod = getattr(module, modName)()
mod.__name__ = modName
# Module may have been renamed or removed
- if modName not in self.ts.config['__modules__']:
+ if modName not in self.config['__modules__']:
continue
# Set up the module
# Configuration is a combined global config with module-specific options
- self.ts.modconfig[modName] = deepcopy(self.ts.config['__modules__'][modName]['opts'])
- for opt in list(self.ts.config.keys()):
- self.ts.modconfig[modName][opt] = deepcopy(self.ts.config[opt])
+ self.modconfig[modName] = deepcopy(self.config['__modules__'][modName]['opts'])
+ for opt in list(self.config.keys()):
+ self.modconfig[modName][opt] = deepcopy(self.config[opt])
mod.clearListeners() # clear any listener relationships from the past
- mod.setup(self.ts.sf, self.ts.modconfig[modName])
- mod.setDbh(self.ts.dbh)
- mod.setScanId(self.ts.scanId)
+ mod.setup(self.sf, self.modconfig[modName])
+ mod.setDbh(self.dbh)
+ mod.setScanId(self.scanId)
# Give modules a chance to 'enrich' the original target with
# aliases of that target.
newTarget = mod.enrichTarget(target)
if newTarget is not None:
target = newTarget
- self.ts.moduleInstances[modName] = mod
+ self.moduleInstances[modName] = mod
# Override the module's local socket module
# to be the SOCKS one.
- if self.ts.config['_socks1type'] != '':
+ if self.config['_socks1type'] != '':
mod._updateSocket(socket)
# Set up event output filters if requested
- if self.ts.config['__outputfilter']:
- mod.setOutputFilter(self.ts.config['__outputfilter'])
+ if self.config['__outputfilter']:
+ mod.setOutputFilter(self.config['__outputfilter'])
- self.ts.sf.status(modName + " module loaded.")
+ self.sf.status(modName + " module loaded.")
# Register listener modules and then start all modules sequentially
- for module in list(self.ts.moduleInstances.values()):
+ for module in list(self.moduleInstances.values()):
# Register the target with the module
module.setTarget(target)
- for listenerModule in list(self.ts.moduleInstances.values()):
+ for listenerModule in list(self.moduleInstances.values()):
# Careful not to register twice or you will get duplicate events
if listenerModule in module._listenerModules:
continue
@@ -222,21 +223,21 @@ class SpiderFootScanner(threading.Thread):
psMod.__name__ = "SpiderFoot UI"
psMod.setTarget(target)
psMod.clearListeners()
- for mod in list(self.ts.moduleInstances.values()):
+ for mod in list(self.moduleInstances.values()):
if mod.watchedEvents() is not None:
psMod.registerListener(mod)
# Create the "ROOT" event which un-triggered modules will link events to
- rootEvent = SpiderFootEvent("ROOT", self.ts.targetValue, "", None)
+ rootEvent = SpiderFootEvent("ROOT", self.targetValue, "", None)
psMod.notifyListeners(rootEvent)
- firstEvent = SpiderFootEvent(self.ts.targetType, self.ts.targetValue,
+ firstEvent = SpiderFootEvent(self.targetType, self.targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
# Special case.. check if an INTERNET_NAME is also a domain
- if self.ts.targetType == 'INTERNET_NAME':
- if self.ts.sf.isDomain(self.ts.targetValue, self.ts.config['_internettlds']):
- firstEvent = SpiderFootEvent('DOMAIN_NAME', self.ts.targetValue,
+ if self.targetType == 'INTERNET_NAME':
+ if self.sf.isDomain(self.targetValue, self.config['_internettlds']):
+ firstEvent = SpiderFootEvent('DOMAIN_NAME', self.targetValue,
"SpiderFoot UI", rootEvent)
psMod.notifyListeners(firstEvent)
@@ -246,26 +247,24 @@ class SpiderFootScanner(threading.Thread):
# Check in case the user requested to stop the scan between modules
# initializing
- for module in list(self.ts.moduleInstances.values()):
+ for module in list(self.moduleInstances.values()):
if module.checkForStop():
self.setStatus('ABORTING')
aborted = True
break
if aborted:
- self.ts.sf.status("Scan [" + self.ts.scanId + "] aborted.")
+ self.sf.status("Scan [" + self.scanId + "] aborted.")
self.setStatus("ABORTED", None, time.time() * 1000)
else:
- self.ts.sf.status("Scan [" + self.ts.scanId + "] completed.")
+ self.sf.status("Scan [" + self.scanId + "] completed.")
self.setStatus("FINISHED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
- self.ts.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
+ self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
"encountered during scan. Please report this as a bug: " + \
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
- self.ts.sf.status("Scan [" + self.ts.scanId + "] failed: " + str(e))
+ self.sf.status("Scan [" + self.scanId + "] failed: " + str(e))
self.setStatus("ERROR-FAILED", None, time.time() * 1000)
- self.ts.dbh.close()
- del self.ts
- del self.temp
+ self.dbh.close()
\ No newline at end of file
diff --git a/sfwebui.py b/sfwebui.py
index 4a7c7183..6a65de21 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -15,6 +15,7 @@ import html
import csv
import time
import random
+import multiprocessing as mp
from secure import SecureHeaders
from cherrypy import _cperror
from operator import itemgetter
@@ -25,6 +26,7 @@ from sfdb import SpiderFootDb
from sflib import SpiderFoot, globalScanStatus
from sfscan import SpiderFootScanner
from io import StringIO
+mp.set_start_method("spawn", force=True)
class SpiderFootWebUi:
@@ -828,14 +830,14 @@ class SpiderFootWebUi:
scantarget = scantarget.replace("\"", "")
else:
scantarget = scantarget.lower()
- t = SpiderFootScanner(scanname, scantarget, targetType, scanId,
- modlist, cfg, modopts)
- t.start()
+ p = mp.Process(target=SpiderFootScanner, args=(scanname, scantarget, targetType, scanId,
+ modlist, cfg, modopts))
+ p.start()
# Wait until the scan has initialized
- while globalScanStatus.getStatus(scanId) is None:
- print("[info] Waiting for the scan to initialize...")
- time.sleep(1)
+ # while globalScanStatus.getStatus(scanId) is None:
+ # print("[info] Waiting for the scan to initialize...")
+ # time.sleep(1)
if not cli:
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)