Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Proper locking on threaded modules.

smicallefa711141d75e8
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 57de6d4d..4934f321 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -45,6 +45,7 @@ class sfp_accounts(SpiderFootPlugin):
sites = list()
errorState = False
distrustedChecked = False
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
@@ -54,6 +55,7 @@ class sfp_accounts(SpiderFootPlugin):
self.errorState = False
self.distrustedChecked = False
self.__dataSource__ = "Social Media"
+ self.lock = threading.Lock()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -98,11 +100,13 @@ class sfp_accounts(SpiderFootPlugin):
useragent=self.opts['_useragent'], noLog=True)
if not res['content']:
- self.siteResults[retname] = False
+ with self.lock:
+ self.siteResults[retname] = False
return
if res['code'].startswith("4") or res['code'].startswith("5"):
- self.siteResults[retname] = False
+ with self.lock:
+ self.siteResults[retname] = False
return
try:
@@ -135,7 +139,8 @@ class sfp_accounts(SpiderFootPlugin):
if firstname + "<" in res['content'] or firstname + '"' in res['content']:
found = False
- self.siteResults[retname] = found
+ with self.lock:
+ self.siteResults[retname] = found
def threadSites(self, name, siteList):
ret = list()
diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.py
index 427696f7..4d299e42 100644
--- a/modules/sfp_azureblobstorage.py
+++ b/modules/sfp_azureblobstorage.py
@@ -32,6 +32,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):
results = None
s3results = None
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.py
index 7a2daf58..ef23c12a 100644
--- a/modules/sfp_digitaloceanspace.py
+++ b/modules/sfp_digitaloceanspace.py
@@ -34,11 +34,13 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
results = None
s3results = None
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.s3results = dict()
self.results = dict()
+ self.lock = threading.Lock()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -60,10 +62,11 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
(res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
- if "ListBucketResult" in res['content']:
- self.s3results[url] = res['content'].count("<Key>")
- else:
- self.s3results[url] = 0
+ with self.lock:
+ if "ListBucketResult" in res['content']:
+ self.s3results[url] = res['content'].count("<Key>")
+ else:
+ self.s3results[url] = 0
def threadSites(self, siteList):
ret = list()
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.py
index 9cc05aa7..82bb3c5a 100644
--- a/modules/sfp_dnsbrute.py
+++ b/modules/sfp_dnsbrute.py
@@ -50,6 +50,7 @@ class sfp_dnsbrute(SpiderFootPlugin):
events = dict()
resolveCache = dict()
sublist = dict()
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
@@ -57,6 +58,7 @@ class sfp_dnsbrute(SpiderFootPlugin):
self.events = dict()
self.resolveCache = dict()
self.__dataSource__ = "DNS"
+ self.lock = threading.Lock()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -104,9 +106,11 @@ class sfp_dnsbrute(SpiderFootPlugin):
name = name.encode("idna")
addrs = resolver.query(name)
- self.hostResults[name] = True
+ with self.lock:
+ self.hostResults[name] = True
except BaseException as e:
- self.hostResults[name] = False
+ with self.lock:
+ self.hostResults[name] = False
def tryHostWrapper(self, hostList, sourceEvent):
self.hostResults = dict()
diff --git a/modules/sfp_portscan_tcp.py b/modules/sfp_portscan_tcp.py
index 8e792823..82117e26 100644
--- a/modules/sfp_portscan_tcp.py
+++ b/modules/sfp_portscan_tcp.py
@@ -52,11 +52,13 @@ class sfp_portscan_tcp(SpiderFootPlugin):
results = dict()
portlist = list()
portResults = dict()
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = dict()
self.__dataSource__ = "Target Network"
+ self.lock = threading.Lock()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -88,14 +90,17 @@ class sfp_portscan_tcp(SpiderFootPlugin):
try:
sock = socket.create_connection((ip, port), self.opts['timeout'])
sock.settimeout(self.opts['timeout'])
- self.portResults[ip + ":" + str(port)] = True
+ with self.lock:
+ self.portResults[ip + ":" + str(port)] = True
except Exception as e:
- self.portResults[ip + ":" + str(port)] = False
+ with self.lock:
+ self.portResults[ip + ":" + str(port)] = False
return
# If the port was open, see what we can read
try:
- self.portResults[ip + ":" + str(port)] = sock.recv(4096)
+ with self.lock:
+ self.portResults[ip + ":" + str(port)] = sock.recv(4096)
except Exception as e:
sock.close()
return
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
index 167d6632..ee63b11b 100644
--- a/modules/sfp_s3bucket.py
+++ b/modules/sfp_s3bucket.py
@@ -34,11 +34,13 @@ class sfp_s3bucket(SpiderFootPlugin):
results = list()
s3results = dict()
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.s3results = dict()
self.results = list()
+ self.lock = threading.Lock()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -62,9 +64,11 @@ class sfp_s3bucket(SpiderFootPlugin):
return None
else:
if "ListBucketResult" in res['content']:
- self.s3results[url] = res['content'].count("<Key>")
+ with self.lock:
+ self.s3results[url] = res['content'].count("<Key>")
else:
- self.s3results[url] = 0
+ with self.lock:
+ self.s3results[url] = 0
def threadSites(self, siteList):
ret = list()
diff --git a/modules/sfp_tldsearch.py b/modules/sfp_tldsearch.py
index 58a5b4c0..b5e6dc48 100644
--- a/modules/sfp_tldsearch.py
+++ b/modules/sfp_tldsearch.py
@@ -40,11 +40,13 @@ class sfp_tldsearch(SpiderFootPlugin):
# Track TLD search results between threads
tldResults = dict()
+ lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = list()
self.__dataSource__ = "DNS"
+ self.lock = threading.Lock()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -64,9 +66,11 @@ class sfp_tldsearch(SpiderFootPlugin):
return None
try:
addrs = socket.gethostbyname_ex(target)
- self.tldResults[target] = True
+ with self.lock:
+ self.tldResults[target] = True
except BaseException as e:
- self.tldResults[target] = False
+ with self.lock:
+ self.tldResults[target] = False
def tryTldWrapper(self, tldList, sourceEvent):
self.tldResults = dict()