SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Proper locking on threaded modules.
smicallefa711141d75e8
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex 57de6d4d..4934f321 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -45,6 +45,7 @@ class sfp_accounts(SpiderFootPlugin):sites = list()errorState = FalsedistrustedChecked = False+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc@@ -54,6 +55,7 @@ class sfp_accounts(SpiderFootPlugin):self.errorState = Falseself.distrustedChecked = Falseself.__dataSource__ = "Social Media"+ self.lock = threading.Lock()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -98,11 +100,13 @@ class sfp_accounts(SpiderFootPlugin):useragent=self.opts['_useragent'], noLog=True)if not res['content']:- self.siteResults[retname] = False+ with self.lock:+ self.siteResults[retname] = Falsereturnif res['code'].startswith("4") or res['code'].startswith("5"):- self.siteResults[retname] = False+ with self.lock:+ self.siteResults[retname] = Falsereturntry:@@ -135,7 +139,8 @@ class sfp_accounts(SpiderFootPlugin):if firstname + "<" in res['content'] or firstname + '"' in res['content']:found = False- self.siteResults[retname] = found+ with self.lock:+ self.siteResults[retname] = founddef threadSites(self, name, siteList):ret = list()diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.pyindex 427696f7..4d299e42 100644--- a/modules/sfp_azureblobstorage.py+++ b/modules/sfp_azureblobstorage.py@@ -32,6 +32,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):results = Nones3results = None+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcdiff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.pyindex 7a2daf58..ef23c12a 100644--- a/modules/sfp_digitaloceanspace.py+++ b/modules/sfp_digitaloceanspace.py@@ -34,11 +34,13 @@ class sfp_digitaloceanspace(SpiderFootPlugin):results = Nones3results = None+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.s3results = dict()self.results = dict()+ self.lock = threading.Lock()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -60,10 +62,11 @@ class sfp_digitaloceanspace(SpiderFootPlugin):(res['content'] is None or "NoSuchBucket" in res['content']):self.sf.debug("Not a valid bucket: " + url)else:- if "ListBucketResult" in res['content']:- self.s3results[url] = res['content'].count("<Key>")- else:- self.s3results[url] = 0+ with self.lock:+ if "ListBucketResult" in res['content']:+ self.s3results[url] = res['content'].count("<Key>")+ else:+ self.s3results[url] = 0def threadSites(self, siteList):ret = list()diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.pyindex 9cc05aa7..82bb3c5a 100644--- a/modules/sfp_dnsbrute.py+++ b/modules/sfp_dnsbrute.py@@ -50,6 +50,7 @@ class sfp_dnsbrute(SpiderFootPlugin):events = dict()resolveCache = dict()sublist = dict()+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc@@ -57,6 +58,7 @@ class sfp_dnsbrute(SpiderFootPlugin):self.events = dict()self.resolveCache = dict()self.__dataSource__ = "DNS"+ self.lock = threading.Lock()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -104,9 +106,11 @@ class sfp_dnsbrute(SpiderFootPlugin):name = name.encode("idna")addrs = resolver.query(name)- self.hostResults[name] = True+ with self.lock:+ self.hostResults[name] = Trueexcept BaseException as e:- self.hostResults[name] = False+ with self.lock:+ self.hostResults[name] = Falsedef tryHostWrapper(self, hostList, sourceEvent):self.hostResults = dict()diff --git a/modules/sfp_portscan_tcp.py b/modules/sfp_portscan_tcp.pyindex 8e792823..82117e26 100644--- a/modules/sfp_portscan_tcp.py+++ b/modules/sfp_portscan_tcp.py@@ -52,11 +52,13 @@ class sfp_portscan_tcp(SpiderFootPlugin):results = dict()portlist = list()portResults = dict()+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = dict()self.__dataSource__ = "Target Network"+ self.lock = threading.Lock()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -88,14 +90,17 @@ class sfp_portscan_tcp(SpiderFootPlugin):try:sock = socket.create_connection((ip, port), self.opts['timeout'])sock.settimeout(self.opts['timeout'])- self.portResults[ip + ":" + str(port)] = True+ with self.lock:+ self.portResults[ip + ":" + str(port)] = Trueexcept Exception as e:- self.portResults[ip + ":" + str(port)] = False+ with self.lock:+ self.portResults[ip + ":" + str(port)] = Falsereturn# If the port was open, see what we can readtry:- self.portResults[ip + ":" + str(port)] = sock.recv(4096)+ with self.lock:+ self.portResults[ip + ":" + str(port)] = sock.recv(4096)except Exception as e:sock.close()returndiff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.pyindex 167d6632..ee63b11b 100644--- a/modules/sfp_s3bucket.py+++ b/modules/sfp_s3bucket.py@@ -34,11 +34,13 @@ class sfp_s3bucket(SpiderFootPlugin):results = list()s3results = dict()+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.s3results = dict()self.results = list()+ self.lock = threading.Lock()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -62,9 +64,11 @@ class sfp_s3bucket(SpiderFootPlugin):return Noneelse:if "ListBucketResult" in res['content']:- self.s3results[url] = res['content'].count("<Key>")+ with self.lock:+ self.s3results[url] = res['content'].count("<Key>")else:- self.s3results[url] = 0+ with self.lock:+ self.s3results[url] = 0def threadSites(self, siteList):ret = list()diff --git a/modules/sfp_tldsearch.py b/modules/sfp_tldsearch.pyindex 58a5b4c0..b5e6dc48 100644--- a/modules/sfp_tldsearch.py+++ b/modules/sfp_tldsearch.py@@ -40,11 +40,13 @@ class sfp_tldsearch(SpiderFootPlugin):# Track TLD search results between threadstldResults = dict()+ lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = list()self.__dataSource__ = "DNS"+ self.lock = threading.Lock()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -64,9 +66,11 @@ class sfp_tldsearch(SpiderFootPlugin):return Nonetry:addrs = socket.gethostbyname_ex(target)- self.tldResults[target] = True+ with self.lock:+ self.tldResults[target] = Trueexcept BaseException as e:- self.tldResults[target] = False+ with self.lock:+ self.tldResults[target] = Falsedef tryTldWrapper(self, tldList, sourceEvent):self.tldResults = dict()