SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Module clean-ups, tons of bug fixes and tweaks.
smicallef38985a6dbec4
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.pyindex b9a323aa..5f193b9a 100644--- a/modules/sfp__stor_stdout.py+++ b/modules/sfp__stor_stdout.py@@ -85,7 +85,6 @@ class sfp__stor_stdout(SpiderFootPlugin):d['type'] = self.opts['_eventtypes'][event.eventType]print(json.dumps(d))-# Handle events sent to this moduledef handleEvent(self, sfEvent):if sfEvent.eventType == "ROOT":diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex acbb6c8f..ebd83ab9 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -157,7 +157,7 @@ class sfp_accounts(SpiderFootPlugin):self.sf.info("Spawning thread to check site: " + site['name'] + \" / " + site['check_uri'].format(account=name))- t.append(threading.Thread(name='sfp_accounts_' + site['name'],+ t.append(threading.Thread(name='thread_sfp_accounts_' + site['name'],target=self.checkSite, args=(name, site)))t[i].start()i += 1@@ -166,7 +166,7 @@ class sfp_accounts(SpiderFootPlugin):while running:found = Falsefor rt in threading.enumerate():- if rt.name.startswith("sfp_accounts_"):+ if rt.name.startswith("thread_sfp_accounts_"):found = Trueif not found:diff --git a/modules/sfp_adblock.py b/modules/sfp_adblock.pyindex 2f2e1d35..ddcb8bfd 100644--- a/modules/sfp_adblock.py+++ b/modules/sfp_adblock.py@@ -27,13 +27,13 @@ class sfp_adblock(SpiderFootPlugin):"blocklist": "AdBlockPlus block list."}- results = list()+ results = Nonerules = NoneerrorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()self.rules = Noneself.errorState = False@@ -81,7 +81,7 @@ class sfp_adblock(SpiderFootPlugin):pagetype = "_INTERNAL"if eventData not in self.results:- self.results.append(eventData)+ self.results[eventData] = Trueelse:self.sf.debug("Already checked this page for AdBlock matching, skipping.")return Nonediff --git a/modules/sfp_ahmia.py b/modules/sfp_ahmia.pyindex c7a6b9a1..9f4db495 100644--- a/modules/sfp_ahmia.py+++ b/modules/sfp_ahmia.py@@ -72,14 +72,10 @@ class sfp_ahmia(SpiderFootPlugin):if self.checkForStop():return None- # Submit the search results for analysis- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],- self.__name__, event)- self.notifyListeners(evt)-links = re.findall("redirect_url=(.[^\"]+)\"",data['content'], re.IGNORECASE | re.DOTALL)+ reported = Falsefor link in links:if link in self.results:continue@@ -110,14 +106,21 @@ class sfp_ahmia(SpiderFootPlugin):self.sf.debug("String not found in content.")continue- data = res['content'][startIndex:endIndex]- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",+ wdata = res['content'][startIndex:endIndex]+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + wdata + "...",self.__name__, evt)self.notifyListeners(evt)-+ reported = Trueelse:evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)self.notifyListeners(evt)+ reported = True++ if reported:+ # Submit the search results for analysis+ evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],+ self.__name__, event)+ self.notifyListeners(evt)# End of sfp_ahmia classdiff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.pyindex 33b436cf..b4b57cc4 100644--- a/modules/sfp_archiveorg.py+++ b/modules/sfp_archiveorg.py@@ -47,12 +47,12 @@ class sfp_archiveorg(SpiderFootPlugin):"javascriptpages": "Query the Wayback Machine for historic versions of URLs using Javascript."}- results = list()+ results = NonefoundDates = list()def setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()self.foundDates = list()for opt in userOpts.keys():@@ -104,7 +104,7 @@ class sfp_archiveorg(SpiderFootPlugin):if eventData in self.results:return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Truefor daysback in self.opts['farback'].split(","):newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.pyindex 4d299e42..36d95f14 100644--- a/modules/sfp_azureblobstorage.py+++ b/modules/sfp_azureblobstorage.py@@ -32,7 +32,6 @@ class sfp_azureblobstorage(SpiderFootPlugin):results = Nones3results = None- lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc@@ -72,7 +71,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='sfp_azureblobstorages_' + site,+ t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + site,target=self.checkSite, args=(site,)))t[i].start()i += 1@@ -81,7 +80,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):while running:found = Falsefor rt in threading.enumerate():- if rt.name.startswith("sfp_azureblobstorages_"):+ if rt.name.startswith("thread_sfp_azureblobstorages_"):found = Trueif not found:diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.pyindex bbf594b5..5e9ff2b0 100644--- a/modules/sfp_bitcoin.py+++ b/modules/sfp_bitcoin.py@@ -11,10 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------try:- import re2 as re-except ImportError:- import re+import refrom hashlib import sha256from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent@@ -24,11 +21,11 @@ class sfp_bitcoin(SpiderFootPlugin):# Default optionsopts = {}- results = list()+ results = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -69,7 +66,7 @@ class sfp_bitcoin(SpiderFootPlugin):if sourceData in self.results:return Noneelse:- self.results.append(sourceData)+ self.results[sourceData] = Trueself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)diff --git a/modules/sfp_company.py b/modules/sfp_company.pyindex 49c964cc..bf1d8a46 100644--- a/modules/sfp_company.py+++ b/modules/sfp_company.py@@ -11,17 +11,12 @@# Licence: GPL# --------------------------------------------------------------------------------try:- import re2 as re-except ImportError:- import re-+import refrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_company(SpiderFootPlugin):"""Company Names:Footprint,Investigate,Passive:Content Analysis::Identify company names in any obtained data."""-# Default optionsopts = {# options specific to this module@@ -85,7 +80,7 @@ class sfp_company(SpiderFootPlugin):return Noneif eventName == "TARGET_WEB_CONTENT":- url = event.sourceEvent.data+ url = event.actualSourceif self.opts['filterjscss'] and (".js" in url or ".css" in url):self.sf.debug("Ignoring web content from CSS/JS.")return Nonediff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.pyindex 3958c136..401f4435 100644--- a/modules/sfp_cookie.py+++ b/modules/sfp_cookie.py@@ -10,9 +10,9 @@# Licence: GPL# -------------------------------------------------------------------------------+import jsonfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-class sfp_cookie(SpiderFootPlugin):"""Cookies:Footprint,Investigate,Passive:Crawling and Scanning::Extract Cookies from HTTP headers."""@@ -43,8 +43,7 @@ class sfp_cookie(SpiderFootPlugin):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data- parentEvent = event.sourceEvent- eventSource = event.sourceEvent.data+ eventSource = event.actualSourceself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)if eventSource in self.results:@@ -56,9 +55,17 @@ class sfp_cookie(SpiderFootPlugin):self.sf.debug("Not collecting cookies from external sites.")return None- if 'set-cookie' in eventData:- evt = SpiderFootEvent("TARGET_WEB_COOKIE", eventData['set-cookie'],- self.__name__, parentEvent)+ try:+ jdata = json.loads(eventData)+ if jdata == None:+ return None+ except BaseException as e:+ self.sf.error("Received HTTP headers from another module in an unexpected format.", False)+ return None++ if 'set-cookie' in jdata:+ evt = SpiderFootEvent("TARGET_WEB_COOKIE", jdata['set-cookie'],+ self.__name__, event)self.notifyListeners(evt)# End of sfp_cookie classdiff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.pyindex fa3035b3..0de2ea19 100644--- a/modules/sfp_crossref.py+++ b/modules/sfp_crossref.py@@ -16,11 +16,9 @@import refrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-class sfp_crossref(SpiderFootPlugin):"""Cross-Reference:Footprint:Crawling and Scanning::Identify whether other domains are associated ('Affiliates') of the target."""-# Default optionsopts = {'checkbase': True@@ -32,21 +30,18 @@ class sfp_crossref(SpiderFootPlugin):}# Internal results tracking- results = None- fetched = list()+ fetched = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = self.tempStorage()- self.fetched = list()+ self.fetched = self.tempStorage()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]# What events is this module interested in for inputdef watchedEvents(self):- return ['LINKED_URL_EXTERNAL', 'SIMILARDOMAIN',- 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL']+ return ['LINKED_URL_EXTERNAL', 'SIMILARDOMAIN', 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL']# What events this module produces# This is to support the end user in selecting modules based on events@@ -78,11 +73,11 @@ class sfp_crossref(SpiderFootPlugin):self.sf.debug("Ignoring " + eventData + " as already tested")returnelse:- self.fetched.append(eventData)+ self.fetched[eventData] = Trueself.sf.debug("Testing for affiliation: " + eventData)res = self.sf.fetchUrl(eventData, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ useragent=self.opts['_useragent'], sizeLimit=10000000)if res['content'] is None:self.sf.debug("Ignoring " + eventData + " as no data returned")@@ -108,10 +103,11 @@ class sfp_crossref(SpiderFootPlugin):if url in self.fetched:return Noneelse:- self.fetched.append(url)+ self.fetched[url] = Trueres = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ useragent=self.opts['_useragent'],+ sizeLimit=10000000)if res['content'] is not None:for name in self.getTarget().getNames():pat = re.compile("([\.\'\/\"\ ]" + name + "[\'\/\"\ ])",@@ -134,4 +130,3 @@ class sfp_crossref(SpiderFootPlugin):evt2.moduleDataSource = event.moduleDataSourceself.notifyListeners(evt2)-# End of sfp_crossref classdiff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.pyindex c711a2a5..517f7586 100644--- a/modules/sfp_digitaloceanspace.py+++ b/modules/sfp_digitaloceanspace.py@@ -33,7 +33,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):}results = None- s3results = None+ s3results = dict()lock = Nonedef setup(self, sfc, userOpts=dict()):@@ -56,16 +56,17 @@ class sfp_digitaloceanspace(SpiderFootPlugin):return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]def checkSite(self, url):- res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")if res['code'] not in [ "301", "302", "200" ] and \(res['content'] is None or "NoSuchBucket" in res['content']):self.sf.debug("Not a valid bucket: " + url)else:- with self.lock:- if "ListBucketResult" in res['content']:+ if "ListBucketResult" in res['content']:+ with self.lock:self.s3results[url] = res['content'].count("<Key>")- else:+ else:+ with self.lock:self.s3results[url] = 0def threadSites(self, siteList):@@ -80,7 +81,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='sfp_digitaloceanspaces_' + site,+ t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + site,target=self.checkSite, args=(site,)))t[i].start()i += 1@@ -89,7 +90,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):while running:found = Falsefor rt in threading.enumerate():- if rt.name.startswith("sfp_digitaloceanspaces_"):+ if rt.name.startswith("thread_sfp_digitaloceanspaces_"):found = Trueif not found:@@ -163,7 +164,8 @@ class sfp_digitaloceanspace(SpiderFootPlugin):evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)self.notifyListeners(evt)if bucket[2] != "0":- evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ":" + bucket[1] + ": " + bucket[2] + " files found.",+ bucketname = bucket[1].replace("//", "")+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucketname + ":" + bucket[1] + ": " + bucket[2] + " files found.",self.__name__, evt)self.notifyListeners(evt)diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.pyindex 8e2b9ab3..017f82bb 100644--- a/modules/sfp_dnsbrute.py+++ b/modules/sfp_dnsbrute.py@@ -29,30 +29,27 @@ class sfp_dnsbrute(SpiderFootPlugin):"top10000": False,"numbersuffix": True,"numbersuffixlimit": True,- "maxthreads": 20+ "_maxthreads": 100}# Option descriptionsoptdescs = {- 'skipcommonwildcard': "If wildcard DNS is detected, don't attempt brute-forcing.",+ 'skipcommonwildcard': "If wildcard DNS is detected, don't bother brute-forcing.",'domainonly': "Only attempt to brute-force names on domain names, not hostnames (some hostnames are also sub-domains).",'commons': "Try a list of about 750 common hostnames/sub-domains.",'top10000': "Try a further 10,000 common hostnames/sub-domains. Will make the scan much slower.",'numbersuffix': "For any host found, try appending 1, 01, 001, -1, -01, -001, 2, 02, etc. (up to 10)",- 'numbersuffixlimit': "Limit using the number suffixes for hosts that have already been resolved? If disabled this will significantly extend the duration of scans.",- 'maxthreads': "Maximum number of concurrent resolution attempts."+ 'numbersuffixlimit': "Limit using the number suffixes for hosts that have already been resolved? If disabled this will significantly extend the duration of scans."}- events = dict()- resolveCache = dict()- sublist = dict()+ events = None+ sublist = Nonelock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.sublist = dict()- self.events = dict()- self.resolveCache = dict()+ self.sublist = self.tempStorage()+ self.events = self.tempStorage()self.__dataSource__ = "DNS"self.lock = threading.Lock()@@ -117,7 +114,7 @@ class sfp_dnsbrute(SpiderFootPlugin):# Spawn threads for scanningself.sf.info("Spawning threads to check hosts: " + str(hostList))for name in hostList:- tn = 'sfp_dnsbrute_' + str(random.SystemRandom().randint(0, 999999999))+ tn = 'thread_sfp_dnsbrute_' + str(random.randint(0, 999999999))t.append(threading.Thread(name=tn, target=self.tryHost, args=(name,)))t[i].start()i += 1@@ -126,13 +123,13 @@ class sfp_dnsbrute(SpiderFootPlugin):while running:found = Falsefor rt in threading.enumerate():- if rt.name.startswith("sfp_dnsbrute_"):+ if rt.name.startswith("thread_sfp_dnsbrute_"):found = Trueif not found:running = False- time.sleep(0.25)+ time.sleep(0.05)for res in self.hostResults:if self.hostResults.get(res, False):@@ -160,10 +157,14 @@ class sfp_dnsbrute(SpiderFootPlugin):self.events[eventDataHash] = True# Handle Unicode characters in the name- if type(eventData) != unicode:- eventData = unicode(eventData, "utf-8", errors='replace').encode("idna")- else:- eventData = eventData.encode("idna")+ try:+ if type(eventData) != unicode:+ eventData = unicode(eventData, "utf-8", errors='replace').encode("idna")+ else:+ eventData = eventData.encode("idna")+ except BaseException as e:+ self.sf.debug("IDNA-parsing issue for " + eventData + ": " + str(e))+ return Noneif eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False):if not self.opts['numbersuffix']:@@ -173,10 +174,11 @@ class sfp_dnsbrute(SpiderFootPlugin):return Noneh, dom = eventData.split(".", 1)+# Try resolving common nameswildcard = self.sf.checkDnsWildcard(dom)if self.opts['skipcommonwildcard'] and wildcard:- self.sf.debug("Wildcard DNS detected.")+ self.sf.debug("Wildcard DNS detected on " + dom + " so skipping host iteration.")return Nonedom = "." + dom@@ -212,7 +214,7 @@ class sfp_dnsbrute(SpiderFootPlugin):name = sub + "." + eventData- if len(targetList) <= self.opts['maxthreads']:+ if len(targetList) <= self.opts['_maxthreads']:targetList.append(name)else:self.tryHostWrapper(targetList, event)@@ -237,7 +239,7 @@ class sfp_dnsbrute(SpiderFootPlugin):nextsubs[s + "-0" + str(i) + dom] = Truenextsubs[s + "-00" + str(i) + dom] = True- if len(nextsubs.keys()) >= self.opts['maxthreads']:+ if len(nextsubs.keys()) >= self.opts['_maxthreads']:self.tryHostWrapper(nextsubs.keys(), event)nextsubs = dict()diff --git a/modules/sfp_dnscommonsrv.py b/modules/sfp_dnscommonsrv.pyindex 32b07522..47d04149 100644--- a/modules/sfp_dnscommonsrv.py+++ b/modules/sfp_dnscommonsrv.py@@ -24,7 +24,7 @@ class sfp_dnscommonsrv(SpiderFootPlugin):# Option descriptionsoptdescs = {}- events = dict()+ events = Nonecommonsrv = [ # LDAP/Kerberos, used for Active Directory# https://technet.microsoft.com/en-us/library/cc961719.aspx@@ -67,7 +67,7 @@ class sfp_dnscommonsrv(SpiderFootPlugin):def setup(self, sfc, userOpts=dict()):self.sf = sfc- self.events = dict()+ self.events = self.tempStorage()self.__dataSource__ = "DNS"# What events is this module interested in for inputdiff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.pyindex e7b30e2a..fa2aca49 100644--- a/modules/sfp_dnsneighbor.py+++ b/modules/sfp_dnsneighbor.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import dnsfrom netaddr import IPAddressfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent@@ -29,15 +30,15 @@ class sfp_dnsneighbor(SpiderFootPlugin):'lookasidecount': "If look-aside is enabled, the number of IPs on each 'side' of the IP to look up"}- events = dict()- domresults = dict()- hostresults = dict()+ events = None+ domresults = None+ hostresults = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.events = dict()- self.domresults = dict()- self.hostresults = dict()+ self.events = self.tempStorage()+ self.domresults = self.tempStorage()+ self.hostresults = self.tempStorage()self.__dataSource__ = "DNS"for opt in userOpts.keys():@@ -51,7 +52,7 @@ class sfp_dnsneighbor(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["IP_ADDRESS", "AFFILIATE_IPADDR"]+ return ["AFFILIATE_IPADDR", "IP_ADDRESS"]# Handle events sent to this moduledef handleEvent(self, event):@@ -75,18 +76,22 @@ class sfp_dnsneighbor(SpiderFootPlugin):self.sf.error("Invalid IP address received: " + eventData, False)return None- minip = IPAddress(int(ip) - self.opts['lookasidecount'])- maxip = IPAddress(int(ip) + self.opts['lookasidecount'])+ try:+ minip = IPAddress(int(ip) - self.opts['lookasidecount'])+ maxip = IPAddress(int(ip) + self.opts['lookasidecount'])+ except BaseException as e:+ self.sf.error("Received an invalid IP address: " + eventData, False)+ return None+self.sf.debug("Lookaside max: " + str(maxip) + ", min: " + str(minip))s = int(minip)c = int(maxip)while s <= c:- if self.checkForStop():- return None-sip = str(IPAddress(s))self.sf.debug("Attempting look-aside lookup of: " + sip)+ if self.checkForStop():+ return Noneif sip in self.hostresults or sip == eventData:s += 1@@ -134,17 +139,16 @@ class sfp_dnsneighbor(SpiderFootPlugin):self.processHost(addr, parent, True)s += 1- # Process a host/IP, parentEvent is the event that represents this entitydef processHost(self, host, parentEvent, affiliate=None):parentHash = self.sf.hashstring(parentEvent.data)if host not in self.hostresults:- self.hostresults[host] = list(parentHash)+ self.hostresults[host] = [parentHash]else:if parentHash in self.hostresults[host] or parentEvent.data == host:self.sf.debug("Skipping host, " + host + ", already processed.")return Noneelse:- self.hostresults[host].append(parentHash)+ self.hostresults[host] = self.hostresults[host] + [parentHash]self.sf.debug("Found host: " + host)# If the returned hostname is aliaseed to our@@ -172,6 +176,7 @@ class sfp_dnsneighbor(SpiderFootPlugin):if self.sf.validIP(host):htype = "IP_ADDRESS"+ # If names were found, leave them to sfp_dnsresolve to resolveif not htype:return Nonediff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.pyindex 58b8a8d6..7b13cda2 100644--- a/modules/sfp_dnsresolve.py+++ b/modules/sfp_dnsresolve.py@@ -35,15 +35,15 @@ class sfp_dnsresolve(SpiderFootPlugin):'maxnetblock': "Maximum owned netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"}- events = dict()- domresults = dict()- hostresults = dict()+ events = None+ domresults = None+ hostresults = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.events = dict()- self.domresults = dict()- self.hostresults = dict()+ self.events = self.tempStorage()+ self.domresults = self.tempStorage()+ self.hostresults = self.tempStorage()self.__dataSource__ = "DNS"for opt in userOpts.keys():@@ -52,31 +52,8 @@ class sfp_dnsresolve(SpiderFootPlugin):def enrichTarget(self, target):ret = list()# If it's an IP, get the hostname it reverse resolves to- if target.getType() == "IP_ADDRESS":- ret = self.sf.resolveIP(target.getValue())- if target.getType() == "INTERNET_NAME":- ret = self.sf.resolveHost(target.getValue())- if target.getType() == "NETBLOCK_OWNER":- ret = list()- for addr in IPNetwork(target.getValue()):- ipaddr = str(addr)- if ipaddr.split(".")[3] in ['255', '0']:- continue- if '255' in ipaddr.split("."):- continue- ret.append(ipaddr)-- # Add the reverse-resolved hostnames as aliases too..- names = self.sf.resolveIP(ipaddr)-- if self.opts['validatereverse']:- for host in names:- chk = self.sf.resolveHost(host)- if ipaddr in chk:- ret.append(host)- else:- ret.extend(names)-+ self.sf.info("Identifying aliases for specified target(s)")+ ret = self.sf.resolveTargets(target, self.opts['validatereverse'])if not ret:return target@@ -88,6 +65,7 @@ class sfp_dnsresolve(SpiderFootPlugin):idnahost = host.encode("idna")if idnahost != host:target.setAlias(idnahost, "INTERNET_NAME")+# If the target was a hostname/sub-domain, we can# add the domain as an alias for the target. But# not if the target was an IP or subnet.@@ -95,7 +73,7 @@ class sfp_dnsresolve(SpiderFootPlugin):# dom = self.sf.hostDomain(host, self.opts['_internettlds'])# target.setAlias(dom, "INTERNET_NAME")- self.sf.debug("Aliases identified: " + str(target.getAliases()))+ self.sf.info("Aliases identified: " + str(target.getAliases()))return target@@ -104,7 +82,7 @@ class sfp_dnsresolve(SpiderFootPlugin):return [# Events that need some kind of DNS treatment"CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER",- "IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",+ "IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",# Events that may contain hostnames in their content"TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS","CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAN_WHOIS", "NETBLOCK_WHOIS",@@ -119,10 +97,11 @@ class sfp_dnsresolve(SpiderFootPlugin):# produced.def producedEvents(self):return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",- "AFFILIATE_IPADDR", "DOMAIN_NAME", "IPV6_ADDRESS",+ "AFFILIATE_IPADDR", "DOMAIN_NAME", "IPV6_ADDRESS","DOMAIN_NAME_PARENT", "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN","INTERNET_NAME_UNRESOLVED"]+# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventType@@ -132,8 +111,8 @@ class sfp_dnsresolve(SpiderFootPlugin):addrs = NoneparentEvent = event- # Don't be recursive, and trust that sfp_dnsbrute knows what it's doing- if srcModuleName in [ "sfp_dnsresolve", "sfp_dnsbrute" ]:+ # Don't be recursive+ if srcModuleName in [ "sfp_dnsresolve" ]:return Noneself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)@@ -157,24 +136,43 @@ class sfp_dnsresolve(SpiderFootPlugin):# Search for IPs/hosts in raw data, but obviously nothing this module# already produces, as those things are already entities, not raw data.if eventName not in self.producedEvents():- if type(eventData) in [dict, list]:- eventDataStr = str(eventData)- else:- eventDataStr = eventData- data = urllib2.unquote(eventDataStr)+ data = urllib2.unquote(eventData)for name in self.getTarget().getNames():if self.checkForStop():return None++ offset = data.find(name)+ if offset < 0:+ continue++ if offset == 0:+ offset += len(name)+pat = re.compile("(%..)?([a-zA-Z0-9\-\.]+\." + name + ")", re.IGNORECASE)- matches = re.findall(pat, data)- if matches:- for match in matches:- # Wildcard certs will come in as .blah.blah- if match[1].startswith("."):- m = match[1][1:]- else:- m = match[1]- self.processHost(m, parentEvent, False)+ while offset >= 0:+ offset = data.find(name, offset)+ #print "found at offset: " + str(offset)+ if offset < 0:+ break++ # Get 200 bytes before the name to try and get hostnames+ chunkhost = data[(offset-200):(offset+len(name)+1)]++ try:+ matches = re.findall(pat, chunkhost)+ if matches:+ for match in matches:+ # Wildcard certs will come in as .blah.blah+ if match[1].startswith("."):+ m = match[1][1:]+ else:+ m = match[1]+ self.processHost(m, parentEvent, False)+ except Exception as e:+ self.error("Error applying regex to data (" + str(e) + ")", False)++ offset += len(name)+# Nothing left to do with internal links and raw datareturn None@@ -204,9 +202,9 @@ class sfp_dnsresolve(SpiderFootPlugin):if self.checkForStop():return None- addrs = self.sf.resolveIP(ipaddr)- if len(addrs) > 0:+ addrs = self.sf.resolveIP(ipaddr)+ if addrs:self.sf.debug("Found a reversed hostname from " + ipaddr +" (" + str(addrs) + ")")for addr in addrs:@@ -250,13 +248,13 @@ class sfp_dnsresolve(SpiderFootPlugin):def processHost(self, host, parentEvent, affiliate=None):parentHash = self.sf.hashstring(parentEvent.data)if host not in self.hostresults:- self.hostresults[host] = list(parentHash)+ self.hostresults[host] = [parentHash]else:if parentHash in self.hostresults[host] or parentEvent.data == host:self.sf.debug("Skipping host, " + host + ", already processed.")return Noneelse:- self.hostresults[host].append(parentHash)+ self.hostresults[host] = self.hostresults[host] + [parentHash]self.sf.debug("Found host: " + host)# If the returned hostname is aliaseed to our@@ -290,7 +288,7 @@ class sfp_dnsresolve(SpiderFootPlugin):if htype.endswith("INTERNET_NAME"):resolved = self.sf.resolveHost(host)if htype == "INTERNET_NAME" and not resolved:- evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host,+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host,self.__name__, parentEvent)self.notifyListeners(evt)return None@@ -298,6 +296,7 @@ class sfp_dnsresolve(SpiderFootPlugin):if not resolved:return None+ # Report the hostif host != parentEvent.data and htype != parentEvent.eventType:evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)self.notifyListeners(evt)@@ -311,11 +310,10 @@ class sfp_dnsresolve(SpiderFootPlugin):# Try obtain the IPv6 addressip6s = self.sf.resolveHost6(host)- if not ip6s:- return None- for ip6 in ip6s:- evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt)- self.notifyListeners(evt6)+ if ip6s:+ for ip6 in ip6s:+ evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt)+ self.notifyListeners(evt6)if htype == "AFFILIATE_INTERNET_NAME":dom = self.sf.hostDomain(host, self.opts['_internettlds'])diff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.pyindex ed0c7604..c5272162 100644--- a/modules/sfp_dnszonexfer.py+++ b/modules/sfp_dnszonexfer.py@@ -26,11 +26,11 @@ class sfp_dnszonexfer(SpiderFootPlugin):optdescs = {}- events = dict()+ events = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.events = dict()+ self.events = self.tempStorage()self.__dataSource__ = "DNS"diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.pyindex f0d1f1ac..94b17352 100644--- a/modules/sfp_duckduckgo.py+++ b/modules/sfp_duckduckgo.py@@ -16,9 +16,6 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_duckduckgo(SpiderFootPlugin):"""DuckDuckGo:Footprint,Investigate,Passive:Search Engines::Query DuckDuckGo's API for descriptive information about your target."""---# Default optionsopts = {"affiliatedomains": True@@ -29,11 +26,11 @@ class sfp_duckduckgo(SpiderFootPlugin):"affiliatedomains": "For affiliates, look up the domain name, not the hostname. This will usually return more meaningful information about the affiliate."}- results = list()+ results = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -64,7 +61,7 @@ class sfp_duckduckgo(SpiderFootPlugin):self.sf.debug("Already did a search for " + eventData + ", skipping.")return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Trueurl = "https://api.duckduckgo.com/?q=" + eventData + "&format=json&pretty=1"res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],diff --git a/modules/sfp_email.py b/modules/sfp_email.pyindex bdf34bf9..b63019f7 100644--- a/modules/sfp_email.py+++ b/modules/sfp_email.py@@ -11,11 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------try:- import re2 as re-except ImportError:- import re-+import refrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_email(SpiderFootPlugin):diff --git a/modules/sfp_errors.py b/modules/sfp_errors.pyindex 392e8eb1..28b3cb52 100644--- a/modules/sfp_errors.py+++ b/modules/sfp_errors.py@@ -10,10 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------try:- import re2 as re-except ImportError:- import re+import refrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent# Taken from Google Dorks on exploit-db.com@@ -28,7 +25,6 @@ regexps = dict({})-class sfp_errors(SpiderFootPlugin):"""Errors:Footprint,Passive:Content Analysis::Identify common error messages in content like SQL errors, etc."""@@ -74,7 +70,8 @@ class sfp_errors(SpiderFootPlugin):if srcModuleName != "sfp_spider":return None- eventSource = event.sourceEvent.data+ eventSource = event.actualSource+self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)if eventSource not in self.results.keys():@@ -94,9 +91,9 @@ class sfp_errors(SpiderFootPlugin):matches = re.findall(pat, eventData)if len(matches) > 0 and regexpGrp not in self.results[eventSource]:self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)- self.results[eventSource].append(regexpGrp)+ self.results[eventSource] = self.results[eventSource] + [regexpGrp]evt = SpiderFootEvent("ERROR_MESSAGE", regexpGrp,- self.__name__, event.sourceEvent)+ self.__name__, event)self.notifyListeners(evt)return Nonediff --git a/modules/sfp_ethereum.py b/modules/sfp_ethereum.pyindex 8b108e47..4df07666 100644--- a/modules/sfp_ethereum.py+++ b/modules/sfp_ethereum.py@@ -11,10 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------try:- import re2 as re-except ImportError:- import re+import refrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_ethereum(SpiderFootPlugin):diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.pyindex 6bfea624..b66b5ec4 100644--- a/modules/sfp_gravatar.py+++ b/modules/sfp_gravatar.py@@ -34,7 +34,7 @@ class sfp_gravatar(SpiderFootPlugin):self.sf = sfcself.__dataSource__ = 'Gravatar'self.results = self.tempStorage()- self.reportedUsers = dict()+ self.reportedUsers = self.tempStorage()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]diff --git a/modules/sfp_hostsfilenet.py b/modules/sfp_hostsfilenet.pyindex a3be9de7..0d686306 100644--- a/modules/sfp_hostsfilenet.py+++ b/modules/sfp_hostsfilenet.py@@ -21,7 +21,6 @@ malchecks = {}}-class sfp_hostsfilenet(SpiderFootPlugin):"""hosts-file.net Malicious Hosts:Investigate,Passive:Reputation Systems::Check if a host/domain is malicious according to hosts-file.net Malicious Hosts."""@@ -44,11 +43,11 @@ class sfp_hostsfilenet(SpiderFootPlugin):# Be sure to completely clear any class variables in setup()# or you run the risk of data persisting between scan runs.- results = list()+ results = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()# Clear / reset any other class member variables here# or you risk them persisting between threads.@@ -89,6 +88,9 @@ class sfp_hostsfilenet(SpiderFootPlugin):else:self.sf.cachePut("sfmal_" + cid, data['content'])+ if type(data['content']) != unicode:+ data['content'] = unicode(data['content'], 'utf-8', errors='replace')+# Check for the domain and the hostnameif targetType == "domain" and "127.0.0.1\t" + targetDom + "\n" in data['content']:self.sf.debug(targetDom + " found in " + check + " list.")@@ -121,7 +123,7 @@ class sfp_hostsfilenet(SpiderFootPlugin):self.sf.debug("Skipping " + eventData + ", already checked.")return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Trueif eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):return Nonediff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.pyindex 68bfaa5c..ab592eb8 100644--- a/modules/sfp_junkfiles.py+++ b/modules/sfp_junkfiles.py@@ -36,16 +36,16 @@ class sfp_junkfiles(SpiderFootPlugin):}results = None- hosts = dict()- skiphosts = dict()- bases = dict()+ hosts = None+ skiphosts = None+ bases = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()- self.hosts = dict()- self.skiphosts = dict()- self.bases = dict()+ self.hosts = self.tempStorage()+ self.skiphosts = self.tempStorage()+ self.bases = self.tempStorage()self.__dataSource__ = "Target Website"for opt in userOpts.keys():diff --git a/modules/sfp_names.py b/modules/sfp_names.pyindex 5639c97d..df521786 100644--- a/modules/sfp_names.py+++ b/modules/sfp_names.py@@ -68,7 +68,7 @@ class sfp_names(SpiderFootPlugin):# If the source event is web content, check if the source URL was javascript# or CSS, in which case optionally ignore it.if eventName == "TARGET_WEB_CONTENT":- url = event.sourceEvent.data+ url = event.actualSourceif self.opts['filterjscss'] and (".js" in url or ".css" in url):self.sf.debug("Ignoring web content from CSS/JS.")return Nonediff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.pyindex 1fe6ed16..ff44c300 100644--- a/modules/sfp_neutrinoapi.py+++ b/modules/sfp_neutrinoapi.py@@ -56,7 +56,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):# https://www.neutrinoapi.com/api/phone-validate/def queryPhoneValidate(self, qry):res = self.sf.fetchUrl('https://neutrinoapi.com/phone-validate',- postData="output-format=json&number=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],+ postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},timeout=self.opts['timeout'], useragent=self.opts['_useragent'])return self.parseApiResponse(res)@@ -65,7 +65,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):# https://www.neutrinoapi.com/api/ip-info/def queryIpInfo(self, qry):res = self.sf.fetchUrl("https://neutrinoapi.com/ip-info",- postData="output-format=json&ip=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},timeout=self.opts['timeout'], useragent=self.opts['_useragent'])return self.parseApiResponse(res)@@ -74,7 +74,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):# https://www.neutrinoapi.com/api/ip-blocklist/def queryIpBlocklist(self, qry):res = self.sf.fetchUrl("https://neutrinoapi.com/ip-blocklist",- postData="output-format=json&ip=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},timeout=self.opts['timeout'], useragent=self.opts['_useragent'])return self.parseApiResponse(res)@@ -83,7 +83,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):# https://www.neutrinoapi.com/api/host-reputation/def queryHostReputation(self, qry):res = self.sf.fetchUrl("https://neutrinoapi.com/host-reputation",- postData="output-format=json&host=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],+ postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},timeout=self.opts['timeout'], useragent=self.opts['_useragent'])return self.parseApiResponse(res)diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.pyindex b605bc8c..ae16a310 100644--- a/modules/sfp_pageinfo.py+++ b/modules/sfp_pageinfo.py@@ -38,6 +38,7 @@ class sfp_pageinfo(SpiderFootPlugin):def setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()+ self.__dataSource__ = "Target Website"for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -65,7 +66,7 @@ class sfp_pageinfo(SpiderFootPlugin):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data- eventSource = event.sourceEvent.data # will be the URL of the raw data+ eventSource = event.actualSourceself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)@@ -75,14 +76,14 @@ class sfp_pageinfo(SpiderFootPlugin):self.sf.debug("Not gathering page info for external site " + eventSource)return None- if eventSource not in self.results.keys():+ if eventSource not in self.results:self.results[eventSource] = list()else:self.sf.debug("Already checked this page for a page type, skipping.")return None# Check the configured regexps to determine the page type- for regexpGrp in regexps.keys():+ for regexpGrp in regexps:if regexpGrp in self.results[eventSource]:continue@@ -91,15 +92,14 @@ class sfp_pageinfo(SpiderFootPlugin):matches = re.findall(rx, eventData)if len(matches) > 0 and regexpGrp not in self.results[eventSource]:self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)- self.results[eventSource].append(regexpGrp)- evt = SpiderFootEvent(regexpGrp, eventSource,- self.__name__, event.sourceEvent)+ self.results[eventSource] = self.results[eventSource] + [regexpGrp]+ evt = SpiderFootEvent(regexpGrp, eventSource, self.__name__, event)self.notifyListeners(evt)# If no regexps were matched, consider this a static pageif len(self.results[eventSource]) == 0:self.sf.info("Treating " + eventSource + " as URL_STATIC")- evt = SpiderFootEvent("URL_STATIC", eventSource, self.__name__, event.sourceEvent)+ evt = SpiderFootEvent("URL_STATIC", eventSource, self.__name__, event)self.notifyListeners(evt)# Check for externally referenced Javascript pages@@ -110,7 +110,7 @@ class sfp_pageinfo(SpiderFootPlugin):if '://' in match and not self.getTarget().matches(self.sf.urlFQDN(match)):self.sf.debug("Externally hosted Javascript found at: " + match)evt = SpiderFootEvent("PROVIDER_JAVASCRIPT", match,- self.__name__, event.sourceEvent)+ self.__name__, event)self.notifyListeners(evt)return Nonediff --git a/modules/sfp_phone.py b/modules/sfp_phone.pyindex cac9c2a1..6e58f4c7 100644--- a/modules/sfp_phone.py+++ b/modules/sfp_phone.py@@ -25,11 +25,11 @@ class sfp_phone(SpiderFootPlugin):# Default optionsopts = {}- results = list()+ results = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -54,7 +54,7 @@ class sfp_phone(SpiderFootPlugin):if sourceData in self.results:return Noneelse:- self.results.append(sourceData)+ self.results[sourceData] = Trueself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)diff --git a/modules/sfp_ripe.py b/modules/sfp_ripe.pyindex 9839267c..8e0d8d3d 100644--- a/modules/sfp_ripe.py+++ b/modules/sfp_ripe.py@@ -25,17 +25,17 @@ class sfp_ripe(SpiderFootPlugin):results = NonecurrentEventSrc = None- memCache = dict()- nbreported = dict()+ memCache = None+ nbreported = Nonekeywords = NonelastContent = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()- self.memCache = dict()+ self.memCache = self.tempStorage()self.currentEventSrc = None- self.nbreported = dict()+ self.nbreported = self.tempStorage()self.lastContent = Nonefor opt in userOpts.keys():@@ -194,8 +194,9 @@ class sfp_ripe(SpiderFootPlugin):# and the string supplied.def findName(self, string):# Simplest check to perform..- if self.getTarget().getValue() in string:- return True+ for n in self.getTarget().getNames():+ if n in string:+ return Trueif self.keywords is None:self.keywords = self.sf.domainKeywords(self.getTarget().getNames(),diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.pyindex ee63b11b..79796a90 100644--- a/modules/sfp_s3bucket.py+++ b/modules/sfp_s3bucket.py@@ -32,14 +32,14 @@ class sfp_s3bucket(SpiderFootPlugin):"suffixes": "List of suffixes to append to domains tried as bucket names"}- results = list()+ results = Nones3results = dict()lock = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.s3results = dict()- self.results = list()+ self.results = self.tempStorage()self.lock = threading.Lock()for opt in userOpts.keys():@@ -56,12 +56,11 @@ class sfp_s3bucket(SpiderFootPlugin):return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]def checkSite(self, url):- res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")if res['code'] not in [ "301", "302", "200" ] and \(res['content'] is None or "NoSuchBucket" in res['content']):- #self.sf.debug("Not a valid bucket: " + url)- return None+ self.sf.debug("Not a valid bucket: " + url)else:if "ListBucketResult" in res['content']:with self.lock:@@ -82,7 +81,7 @@ class sfp_s3bucket(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='sfp_s3buckets_' + site,+ t.append(threading.Thread(name='thread_sfp_s3buckets_' + site,target=self.checkSite, args=(site,)))t[i].start()i += 1@@ -91,7 +90,7 @@ class sfp_s3bucket(SpiderFootPlugin):while running:found = Falsefor rt in threading.enumerate():- if rt.name.startswith("sfp_s3buckets_"):+ if rt.name.startswith("thread_sfp_s3buckets_"):found = Trueif not found:@@ -134,7 +133,7 @@ class sfp_s3bucket(SpiderFootPlugin):if eventData in self.results:return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Trueself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)@@ -165,7 +164,8 @@ class sfp_s3bucket(SpiderFootPlugin):evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)self.notifyListeners(evt)if bucket[2] != "0":- evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ": " + bucket[2] + " files found.",+ bucketname = bucket[1].replace("//", "")+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucketname + ": " + bucket[2] + " files found.",self.__name__, evt)self.notifyListeners(evt)diff --git a/modules/sfp_similar.py b/modules/sfp_similar.pyindex b2a75333..4c037644 100644--- a/modules/sfp_similar.py+++ b/modules/sfp_similar.py@@ -70,11 +70,11 @@ class sfp_similar(SpiderFootPlugin):}# Internal results tracking- results = list()+ results = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()self.__dataSource__ = "DNS"for opt in userOpts.keys():@@ -104,7 +104,7 @@ class sfp_similar(SpiderFootPlugin):if dom in self.results:return Noneelse:- self.results.append(dom)+ self.results[dom] = True# Search for typospos = 0diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex 972e2d05..aab9418b 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -18,11 +18,10 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_spider(SpiderFootPlugin):"""Spider:Footprint,Investigate:Crawling and Scanning:slow:Spidering of web-pages to extract content for searching."""-# Default optionsopts = {'robotsonly': False, # only follow links specified by robots.txt- 'pause': 1, # number of seconds to pause between fetches+ 'pausesec': 0, # number of seconds to pause between fetches'maxpages': 100, # max number of pages to fetch'maxlevels': 3, # max number of levels to traverse within a site'usecookies': True, # Use cookies?@@ -42,8 +41,8 @@ class sfp_spider(SpiderFootPlugin):# Option descriptionsoptdescs = {'robotsonly': "Only follow links specified by robots.txt?",- 'pause': "Number of seconds to pause between fetches.",'usecookies': "Accept and use cookies?",+ 'pausesec': "Number of seconds to pause between page fetches.",'start': "Prepend targets with these until you get a hit, to start spidering.",'maxpages': "Maximum number of pages to fetch per starting point identified.",'maxlevels': "Maximum levels to traverse per starting point (e.g. hostname or link identified by another module) identified.",@@ -59,19 +58,19 @@ class sfp_spider(SpiderFootPlugin):robotsRules = dict()# Pages already fetched- fetchedPages = dict()+ fetchedPages = None# Events for links identified- urlEvents = dict()+ urlEvents = None# Tracked cookies per site- siteCookies = dict()+ siteCookies = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.fetchedPages = dict()- self.urlEvents = dict()- self.siteCookies = dict()+ self.fetchedPages = self.tempStorage()+ self.urlEvents = self.tempStorage()+ self.siteCookies = self.tempStorage()self.__dataSource__ = "Target Website"for opt in userOpts.keys():@@ -93,7 +92,8 @@ class sfp_spider(SpiderFootPlugin):cookies = self.siteCookies[site]# Fetch the contents of the supplied URL (object returned)fetched = self.sf.fetchUrl(url, False, cookies,- self.opts['_fetchtimeout'], self.opts['_useragent'])+ self.opts['_fetchtimeout'], self.opts['_useragent'],+ sizeLimit=10000000)self.fetchedPages[url] = True# Track cookies a site has sent, then send the back in subsquent requests@@ -110,7 +110,7 @@ class sfp_spider(SpiderFootPlugin):self.contentNotify(url, fetched, self.urlEvents[url])if fetched['realurl'] is not None and fetched['realurl'] != url:- self.sf.debug("Redirect of " + url + " to " + fetched['realurl'])+ #self.sf.debug("Redirect of " + url + " to " + fetched['realurl'])# Store the content for the redirect so that it isn't fetched againself.fetchedPages[fetched['realurl']] = True# Notify modules about the new link@@ -144,7 +144,7 @@ class sfp_spider(SpiderFootPlugin):def cleanLinks(self, links):returnLinks = dict()- for link in links.keys():+ for link in links:linkBase = self.sf.urlBaseUrl(link)linkFQDN = self.sf.urlFQDN(link)@@ -210,16 +210,17 @@ class sfp_spider(SpiderFootPlugin):if sendcontent:event = SpiderFootEvent("TARGET_WEB_CONTENT", httpresult['content'],self.__name__, parentEvent)+ event.actualSource = urlself.notifyListeners(event)- if httpresult.get('headers') != None:- event = SpiderFootEvent("WEBSERVER_HTTPHEADERS",- json.dumps(httpresult['headers'], ensure_ascii=False),- self.__name__, parentEvent)- self.notifyListeners(event)+ event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(httpresult['headers'], ensure_ascii=False),+ self.__name__, parentEvent)+ event.actualSource = url+ self.notifyListeners(event)event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']),self.__name__, parentEvent)+ event.actualSource = urlself.notifyListeners(event)if not httpresult.get('headers'):@@ -229,22 +230,25 @@ class sfp_spider(SpiderFootPlugin):if ctype:event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype,self.__name__, parentEvent)+ event.actualSource = urlself.notifyListeners(event)-# Trigger spidering off the following events..# Spidering and search engines provide LINKED_URL_INTERNAL, and DNS lookups# provide INTERNET_NAME.def watchedEvents(self):return ["LINKED_URL_INTERNAL", "INTERNET_NAME"]+ # Don't notify me about events from myself+ def watchOpts(self):+ return [ 'noself' ]+# What events this module produces# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):return ["WEBSERVER_HTTPHEADERS", "HTTP_CODE", "LINKED_URL_INTERNAL",- "LINKED_URL_EXTERNAL", "TARGET_WEB_CONTENT",- "TARGET_WEB_CONTENT_TYPE"]+ "LINKED_URL_EXTERNAL", "TARGET_WEB_CONTENT", "TARGET_WEB_CONTENT_TYPE"]# Some other modules may request we spider thingsdef handleEvent(self, event):@@ -256,21 +260,19 @@ class sfp_spider(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)if eventData in self.urlEvents:- self.sf.debug("Ignoring " + eventName + " as already spidered or is being spidered.")+ self.sf.debug("Ignoring " + eventData + " as already spidered or is being spidered.")return None+ else:+ self.urlEvents[eventData] = event# Don't spider links we find ourselves, obviouslyif eventName == "LINKED_URL_INTERNAL" and "sfp_spider" in srcModuleName:- self.sf.debug("Ignoring " + eventData + ", from self.")return None- self.urlEvents[eventData] = event-# Determine where to start spidering from if it's a INTERNET_NAME eventif eventName == "INTERNET_NAME":for prefix in self.opts['start']:- url = prefix + eventData- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],+ res = self.sf.fetchUrl(prefix + eventData, timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])if res['content'] is not None:spiderTarget = prefix + eventData@@ -325,7 +327,7 @@ class sfp_spider(SpiderFootPlugin):links = dict()# Fetch content from the new links- for link in nextLinks.keys():+ for link in nextLinks:# Always skip links we've already fetchedif (link in self.fetchedPages):self.sf.debug("Already fetched " + link + ", skipping.")@@ -336,7 +338,7 @@ class sfp_spider(SpiderFootPlugin):return Noneself.sf.debug("Fetching fresh content from: " + link)- time.sleep(self.opts['pause'])+ time.sleep(self.opts['pausesec'])freshLinks = self.processUrl(link)if freshLinks is not None:links.update(freshLinks)diff --git a/modules/sfp_sslcert.py b/modules/sfp_sslcert.pyindex cf411dc0..ee8233da 100644--- a/modules/sfp_sslcert.py+++ b/modules/sfp_sslcert.py@@ -80,8 +80,8 @@ class sfp_sslcert(SpiderFootPlugin):# Handle URLs containing port numbersu = urlparse.urlparse(eventData)port = 443- if url.port:- port = url.port+ if u.port:+ port = u.portfqdn = self.sf.urlFQDN(eventData.lower())else:fqdn = eventDatadiff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.pyindex 0c2ed2fd..99b4083c 100644--- a/modules/sfp_strangeheaders.py+++ b/modules/sfp_strangeheaders.py@@ -24,7 +24,6 @@ headers = ["access-control-allow-origin", "accept-ranges", "age", "allow", "cach"x-frame-options", "x-xss-protection", "content-security-policy", "x-content-security-policy","x-webkit-csp", "x-content-type-options", "x-powered-by", "x-ua-compatible"]-class sfp_strangeheaders(SpiderFootPlugin):"""Strange Headers:Footprint,Passive:Content Analysis::Obtain non-standard HTTP headers returned by web servers."""@@ -56,9 +55,7 @@ class sfp_strangeheaders(SpiderFootPlugin):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data- parentEvent = event.sourceEvent- eventSource = event.sourceEvent.data-+ eventSource = event.actualSourceself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)if eventSource in self.results:@@ -82,7 +79,7 @@ class sfp_strangeheaders(SpiderFootPlugin):if key.lower() not in headers:val = key + ": " + jdata[key]evt = SpiderFootEvent("WEBSERVER_STRANGEHEADER", val,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# End of sfp_strangeheaders classdiff --git a/modules/sfp_tldsearch.py b/modules/sfp_tldsearch.pyindex 987b5123..a176127c 100644--- a/modules/sfp_tldsearch.py+++ b/modules/sfp_tldsearch.py@@ -14,28 +14,27 @@import randomimport threadingimport time+import dnsfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-class sfp_tldsearch(SpiderFootPlugin):"""TLD Search:Footprint:DNS:slow:Search all Internet TLDs for domains with the same name as the target (this can be very slow.)"""# Default optionsopts = {- 'activeonly': True, # Only report domains that have content (try to fetch the page)+ 'activeonly': False, # Only report domains that have content (try to fetch the page)'skipwildcards': True,- 'maxthreads': 100+ '_maxthreads': 50}# Option descriptionsoptdescs = {'activeonly': "Only report domains that have content (try to fetch the page)?",- "skipwildcards": "Skip TLDs and sub-TLDs that have wildcard DNS.",- "maxthreads": "Number of simultaneous DNS resolutions to perform at once."+ "skipwildcards": "Skip TLDs and sub-TLDs that have wildcard DNS."}# Internal results tracking- results = list()+ results = None# Track TLD search results between threadstldResults = dict()@@ -43,7 +42,7 @@ class sfp_tldsearch(SpiderFootPlugin):def setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()self.__dataSource__ = "DNS"self.lock = threading.Lock()@@ -61,16 +60,25 @@ class sfp_tldsearch(SpiderFootPlugin):return ["SIMILARDOMAIN"]def tryTld(self, target, tld):+ resolver = dns.resolver.Resolver()+ resolver.timeout = 1+ resolver.lifetime = 1+ resolver.search = list()+if self.opts['skipwildcards'] and self.sf.checkDnsWildcard(tld):return None- addrs = self.sf.resolveHost(target)- if not addrs:+ try:+ addrs = self.sf.resolveHost(target, resolver)+ if not addrs:+ with self.lock:+ self.tldResults[target] = False+ else:+ with self.lock:+ self.tldResults[target] = True+ except BaseException as e:with self.lock:self.tldResults[target] = False- else:- with self.lock:- self.tldResults[target] = Truedef tryTldWrapper(self, tldList, sourceEvent):self.tldResults = dict()@@ -82,7 +90,7 @@ class sfp_tldsearch(SpiderFootPlugin):self.sf.info("Spawning threads to check TLDs: " + str(tldList))for pair in tldList:(domain, tld) = pair- tn = 'sfp_tldsearch_' + str(random.SystemRandom().randint(0, 999999999))+ tn = 'thread_sfp_tldsearch_' + str(random.SystemRandom().randint(0, 999999999))t.append(threading.Thread(name=tn, target=self.tryTld, args=(domain, tld,)))t[i].start()i += 1@@ -91,22 +99,22 @@ class sfp_tldsearch(SpiderFootPlugin):while running:found = Falsefor rt in threading.enumerate():- if rt.name.startswith("sfp_tldsearch_"):+ if rt.name.startswith("thread_sfp_tldsearch_"):found = Trueif not found:running = False- time.sleep(1)+ time.sleep(0.1)- for res in self.tldResults.keys():+ for res in self.tldResults:if self.tldResults[res] and res not in self.results:self.sendEvent(sourceEvent, res)# Store the result internally and notify listening modulesdef sendEvent(self, source, result):self.sf.info("Found a TLD with the target's name: " + result)- self.results.append(result)+ self.results[result] = True# Inform listening modulesif self.opts['activeonly']:@@ -131,7 +139,7 @@ class sfp_tldsearch(SpiderFootPlugin):if eventData in self.results:return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Truekeyword = self.sf.domainKeyword(eventData, self.opts['_internettlds'])self.sf.debug("Keyword extracted from " + eventData + ": " + keyword)@@ -140,7 +148,7 @@ class sfp_tldsearch(SpiderFootPlugin):if keyword in self.results:return Noneelse:- self.results.append(keyword)+ self.results[keyword] = True# Look through all TLDs for the existence of this target keywordfor tld in self.opts['_internettlds']:@@ -163,7 +171,7 @@ class sfp_tldsearch(SpiderFootPlugin):if self.checkForStop():return None- if len(targetList) <= self.opts['maxthreads']:+ if len(targetList) <= self.opts['_maxthreads']:targetList.append([tryDomain, tld])else:self.tryTldWrapper(targetList, event)diff --git a/modules/sfp_webanalytics.py b/modules/sfp_webanalytics.pyindex ae8aaf62..8cf25bd5 100644--- a/modules/sfp_webanalytics.py+++ b/modules/sfp_webanalytics.py@@ -43,7 +43,6 @@ class sfp_webanalytics(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.datasourceData = self.sf.hashstring(eventData)- parentEvent = event.sourceEventif sourceData in self.results:self.sf.debug("Skipping " + eventData + " as already mapped.")@@ -67,7 +66,7 @@ class sfp_webanalytics(SpiderFootPlugin):self.sf.debug("Google Analytics match: " + m)evt = SpiderFootEvent("WEB_ANALYTICS_ID","Google Analytics: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Google AdSense@@ -79,7 +78,7 @@ class sfp_webanalytics(SpiderFootPlugin):self.sf.debug("Google AdSense match: " + m)evt = SpiderFootEvent("WEB_ANALYTICS_ID","Google AdSense: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Google Website Verification@@ -89,7 +88,7 @@ class sfp_webanalytics(SpiderFootPlugin):self.sf.debug("Google Site Verification match: " + m)evt = SpiderFootEvent("WEB_ANALYTICS_ID","Google Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)matches = re.findall(r'<meta name="verify-v1" content="([a-z0-9\-\+_=]{43,44})"', eventData, re.IGNORECASE)@@ -97,7 +96,7 @@ class sfp_webanalytics(SpiderFootPlugin):self.sf.debug("Google Site Verification match: " + m)evt = SpiderFootEvent("WEB_ANALYTICS_ID","Google Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Quantcast@@ -107,7 +106,7 @@ class sfp_webanalytics(SpiderFootPlugin):self.sf.debug("Quantcast match: " + m)evt = SpiderFootEvent("WEB_ANALYTICS_ID","Quantcast: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Ahrefs Site Verification@@ -116,7 +115,7 @@ class sfp_webanalytics(SpiderFootPlugin):self.sf.debug("Ahrefs Site Verification match: " + m)evt = SpiderFootEvent("WEB_ANALYTICS_ID","Ahrefs Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)if eventName == 'DNS_TEXT':@@ -126,7 +125,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Google Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# LogMeIn Domain Verification@@ -135,14 +134,14 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","LogMeIn Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)matches = re.findall(r'logmein-verification-code=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE)for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","LogMeIn Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# DocuSign Domain Verification@@ -151,7 +150,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","DocuSign Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# GlobalSign Site Verification@@ -160,7 +159,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","GlobalSign Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Atlassian Domain Verification@@ -169,7 +168,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Atlassian Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Adobe IDP Site Verification@@ -178,14 +177,14 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Adobe IDP Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)matches = re.findall(r'adobe-idp-site-verification=([a-f0-9]{64})$', eventData.strip(), re.IGNORECASE)for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Adobe IDP Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Adobe Domain Verification@@ -194,7 +193,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Adobe Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Stripe Domain Verification@@ -203,7 +202,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Stripe Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)@@ -213,7 +212,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","TeamViewer SSO Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Aliyun Site Verification@@ -221,7 +220,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Aliyun Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Facebook Domain Verification@@ -230,7 +229,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Facebook Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Citrix Domain Verification@@ -238,7 +237,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Citrix Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Dropbox Domain Verification@@ -247,7 +246,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Dropbox Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Detectify Domain Verification@@ -256,7 +255,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Detectify Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Drift Domain Verification@@ -264,7 +263,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Drift Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Ahrefs Site Verification@@ -273,7 +272,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Ahrefs Site Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Statuspage.io Domain Verification@@ -282,7 +281,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Statuspage Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Zoom.us Domain Verification@@ -291,7 +290,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Zoom.us Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Mail.ru Domain Verification@@ -299,7 +298,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Mail.ru Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Yandex Domain Verification@@ -307,7 +306,7 @@ class sfp_webanalytics(SpiderFootPlugin):for m in matches:evt = SpiderFootEvent("WEB_ANALYTICS_ID","Yandex Domain Verification: " + m,- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)# Brave Ledger Verificationdiff --git a/modules/sfp_webframework.py b/modules/sfp_webframework.pyindex 97f68d0d..612c8c88 100644--- a/modules/sfp_webframework.py+++ b/modules/sfp_webframework.py@@ -25,7 +25,6 @@ regexps = dict({"Wordpress": list(['\/wp-includes\/', '\/wp-content\/'])})-class sfp_webframework(SpiderFootPlugin):"""Web Framework:Footprint,Passive:Content Analysis::Identify the usage of popular web frameworks like jQuery, YUI and others."""@@ -66,18 +65,15 @@ class sfp_webframework(SpiderFootPlugin):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data+ eventSource = event.actualSource# We only want web contentif srcModuleName != "sfp_spider":return None- # If you are processing TARGET_WEB_CONTENT, this is how you would get the- # source of that raw data (e.g. a URL.)- eventSource = event.sourceEvent.data-self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)- if eventSource not in self.results.keys():+ if eventSource not in self.results:self.results[eventSource] = list()# We only want web content for pages on the target site@@ -94,16 +90,11 @@ class sfp_webframework(SpiderFootPlugin):matches = re.findall(pat, eventData)if len(matches) > 0 and regexpGrp not in self.results[eventSource]:self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)- self.results[eventSource].append(regexpGrp)+ self.results[eventSource] = self.results[eventSource] + [regexpGrp]evt = SpiderFootEvent("URL_WEB_FRAMEWORK", regexpGrp,- self.__name__, event.sourceEvent)+ self.__name__, event)self.notifyListeners(evt)return None- # If you intend for this module to act on its own (e.g. not solely rely- # on events from other modules, then you need to have a start() method- # and within that method call self.checkForStop() to see if you've been- # politely asked by the controller to stop your activities (user abort.)-# End of sfp_webframework classdiff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.pyindex 851927ab..fd0f31f0 100644--- a/modules/sfp_webserver.py+++ b/modules/sfp_webserver.py@@ -14,7 +14,6 @@import jsonfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-class sfp_webserver(SpiderFootPlugin):"""Web Server:Footprint,Investigate,Passive:Content Analysis::Obtain web server banners to identify versions of web servers being used."""@@ -47,8 +46,7 @@ class sfp_webserver(SpiderFootPlugin):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data- parentEvent = event.sourceEvent- eventSource = event.sourceEvent.data+ eventSource = event.actualSourceself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)if eventSource in self.results:@@ -72,10 +70,10 @@ class sfp_webserver(SpiderFootPlugin):if 'location' in jdata:if jdata['location'].startswith('http://') or jdata['location'].startswith('https://'):if self.getTarget().matches(self.sf.urlFQDN(jdata['location'])):- evt = SpiderFootEvent('LINKED_URL_INTERNAL', jdata['location'], self.__name__, parentEvent)+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', jdata['location'], self.__name__, event)self.notifyListeners(evt)else:- evt = SpiderFootEvent('LINKED_URL_EXTERNAL', jdata['location'], self.__name__, parentEvent)+ evt = SpiderFootEvent('LINKED_URL_EXTERNAL', jdata['location'], self.__name__, event)self.notifyListeners(evt)# Check CSP header for linked URLs@@ -84,10 +82,10 @@ class sfp_webserver(SpiderFootPlugin):for string in directive.split(' '):if string.startswith('http://') or string.startswith('https://'):if self.getTarget().matches(self.sf.urlFQDN(string)):- evt = SpiderFootEvent('LINKED_URL_INTERNAL', string, self.__name__, parentEvent)+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', string, self.__name__, event)self.notifyListeners(evt)else:- evt = SpiderFootEvent('LINKED_URL_EXTERNAL', string, self.__name__, parentEvent)+ evt = SpiderFootEvent('LINKED_URL_EXTERNAL', string, self.__name__, event)self.notifyListeners(evt)# Could apply some smarts here, for instance looking for certain@@ -96,14 +94,14 @@ class sfp_webserver(SpiderFootPlugin):# and other errors to see what the header looks like.if 'server' in jdata:evt = SpiderFootEvent("WEBSERVER_BANNER", jdata['server'],- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)self.sf.info("Found web server: " + jdata['server'] + " (" + eventSource + ")")if 'x-powered-by' in jdata:evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", jdata['x-powered-by'],- self.__name__, parentEvent)+ self.__name__, event)self.notifyListeners(evt)return None@@ -127,7 +125,7 @@ class sfp_webserver(SpiderFootPlugin):tech = "PHP"if tech is not None:- evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", tech, self.__name__, parentEvent)+ evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", tech, self.__name__, event)self.notifyListeners(evt)# End of sfp_webserver classdiff --git a/sflib.py b/sflib.pyindex f691b374..4766a332 100644--- a/sflib.py+++ b/sflib.py@@ -898,7 +898,7 @@ class SpiderFoot:## Return a normalised resolution or None if not resolved.- def resolveHost(self, host):+ def resolveHost(self, host, resolver=None):try:# IDNA-encode the hostname in case it contains unicodeif type(host) != unicode:@@ -906,9 +906,12 @@ class SpiderFoot:else:host = host.encode("idna")- addrs = self.normalizeDNS(socket.gethostbyname_ex(host))+ if not resolver:+ addrs = self.normalizeDNS(socket.gethostbyname_ex(host))+ else:+ addrs = self.normalizeDNS(resolver.query(host))if len(addrs) > 0:- return addrs+ return set(addrs)return Noneexcept BaseException as e:self.debug("Unable to resolve " + host + ": " + str(e))@@ -921,7 +924,7 @@ class SpiderFoot:try:addrs = self.normalizeDNS(socket.gethostbyaddr(ipaddr))if len(addrs) > 0:- return addrs+ return set(addrs)return Noneexcept BaseException as e:self.debug("Unable to resolve " + ipaddr + " (" + str(e) + ")")@@ -938,7 +941,7 @@ class SpiderFoot:if len(addrs) < 1:return Noneself.debug("Resolved " + hostname + " to IPv6: " + str(addrs))- return addrs+ return set(addrs)except BaseException as e:self.debug("Unable to IPv6 resolve " + hostname + " (" + str(e) + ")")return None@@ -956,6 +959,43 @@ class SpiderFoot:return False+ # Resolve alternative names for a given target+ def resolveTargets(self, target, validateReverse):+ ret = list()+ t = target.getType()+ v = target.getValue()++ if t == "IP_ADDRESS":+ r = self.resolveIP(v)+ if r:+ ret.extend(r)+ if t == "INTERNET_NAME":+ r = self.resolveHost(v)+ if r:+ ret.extend(r)+ if t == "NETBLOCK_OWNER":+ for addr in IPNetwork(v):+ ipaddr = str(addr)+ if ipaddr.split(".")[3] in ['255', '0']:+ continue+ if '255' in ipaddr.split("."):+ continue+ ret.append(ipaddr)++ # Add the reverse-resolved hostnames as aliases too..+ names = self.resolveIP(ipaddr)+ if names:+ if validateReverse:+ for host in names:+ chk = self.resolveHost(host)+ if ipaddr in chk:+ ret.append(host)+ else:+ ret.extend(names)+ if len(ret) > 0:+ return set(ret)+ return None+# Create a safe socket that's using SOCKS/TOR if it was enableddef safeSocket(self, host, port, timeout):sock = socket.create_connection((host, int(port)), int(timeout))@@ -1300,15 +1340,43 @@ class SpiderFoot:if url is None:return None- # Clean the URL- if type(url) != unicode:- url = unicode(url, 'utf-8', errors='replace')+ proxies = dict()+ if self.opts['_socks1type']:+ neverProxyNames = [ self.opts['_socks2addr'] ]+ neverProxySubnets = [ "192.168.", "127.", "10." ]+ host = self.urlFQDN(url)- # Convert any unicode chars in the URL- url = str(self.urlEncodeUnicode(url))+ # Completely on or off?+ if self.opts['_socks1type']:+ proxy = True+ else:+ proxy = False++ # Never proxy these system/internal locations+ # This logic also exists in ext/socks.py and may not be+ # needed anymore.+ if host in neverProxyNames:+ proxy = False+ for s in neverProxyNames:+ if host.endswith(s):+ proxy = False+ for sub in neverProxySubnets:+ if host.startswith(sub):+ proxy = False++ if proxy:+ self.debug("Using proxy for " + host)+ self.debug("Proxy set to " + self.opts['_socks2addr'] + ":" + str(self.opts['_socks3port']))+ proxies = {+ 'http': 'socks5h://' + self.opts['_socks2addr'] + ":" + str(self.opts['_socks3port']),+ 'https': 'socks5h://' + self.opts['_socks2addr'] + ":" + str(self.opts['_socks3port'])+ }+ else:+ self.debug("Not using proxy for " + host)try:header = dict()+ btime = time.time()if type(useragent) is list:header['User-Agent'] = random.SystemRandom().choice(useragent)else:@@ -1317,15 +1385,19 @@ class SpiderFoot:# Add custom headersif headers is not None:for k in headers.keys():- header[k] = headers[k]+ if type(headers[k]) != unicode:+ header[k] = unicode(headers[k], 'utf-8', errors='replace')+ else:+ header[k] = headers[k]if sizeLimit or headOnly:if not noLog:self.info("Fetching (HEAD only): " + url + \- " [timeout: " + \+ " [user-agent: " + header['User-Agent'] + "] [timeout: " + \str(timeout) + "]")- hdr = requests.head(url, headers=header, verify=verify, timeout=timeout)+ hdr = requests.head(url, headers=header, proxies=proxies,+ verify=False, timeout=timeout)size = int(hdr.headers.get('content-length', 0))result['realurl'] = hdr.headers.get('location', url)result['code'] = str(hdr.status_code)@@ -1339,74 +1411,99 @@ class SpiderFoot:if result['realurl'] != url:if not noLog:self.info("Fetching (HEAD only): " + url + \- " [timeout: " + \+ " [user-agent: " + header['User-Agent'] + "] [timeout: " + \str(timeout) + "]")- hdr = requests.head(result['realurl'], headers=header, verify=verify)+ hdr = requests.head(result['realurl'], headers=header, proxies=proxies,+ verify=False, timeout=timeout)size = int(hdr.headers.get('content-length', 0))result['realurl'] = hdr.headers.get('location', result['realurl'])result['code'] = str(hdr.status_code)if size > sizeLimit:return result-- req = urllib2.Request(url, postData, header)if cookies is not None:- req.add_header('cookie', cookies)+ #req.add_header('cookie', cookies)if not noLog:self.info("Fetching (incl. cookies): " + url + \- " [timeout: " + \+ " [user-agent: " + header['User-Agent'] + "] [timeout: " + \str(timeout) + "]")else:if not noLog:- self.info("Fetching: " + url + " [timeout: " + str(timeout) + "]")-+ self.info("Fetching: " + url + " [user-agent: " + \+ header['User-Agent'] + "] [timeout: " + str(timeout) + "]")++ #+ # MAKE THE REQUEST+ #+ if postData:+ res = requests.post(url, data=postData, headers=header, proxies=proxies,+ cookies=cookies, timeout=timeout, verify=False)+ else:+ res = requests.get(url, headers=header, proxies=proxies,+ cookies=cookies, timeout=timeout, verify=False)result['headers'] = dict()- opener = urllib2.build_opener(SmartRedirectHandler())- fullPage = opener.open(req, timeout=timeout)- content = fullPage.read()-- for k, v in fullPage.info().items():- result['headers'][k.lower()] = v+ for h in res.headers:+ if type(h) != unicode:+ hu = unicode(h, 'utf-8', errors='replace')+ else:+ hu = h+ v = res.headers.get(h)+ if type(v) != unicode:+ vu = unicode(v, 'utf-8', errors='replace')+ else:+ vu = v+ result['headers'][hu.lower()] = vu- # Content is compressed- if 'gzip' in result['headers'].get('content-encoding', ''):- content = gzip.GzipFile(fileobj=StringIO.StringIO(content)).read()+ # Sometimes content exceeds the size limit after decompression+ if sizeLimit and len(res.content) > sizeLimit:+ self.debug("Content exceeded size limit, so returning no data just headers")+ result['realurl'] = res.url+ result['code'] = str(res.status_code)+ return result+ if 'refresh' in result['headers']:+ try:+ newurl = result['headers']['refresh'].split(";url=")[1]+ except BaseException as e:+ self.debug("Refresh header found but was not parsable: " + result['headers']['refresh'])+ return result+ self.debug("Refresh header found, re-directing to " + newurl)+ return self.fetchUrl(newurl, fatal, cookies, timeout,+ useragent, headers, noLog, postData,+ dontMangle, sizeLimit, headOnly)++ #print "FOR: " + url+ #print "HEADERS: " + str(result['headers'])+ result['realurl'] = res.url+ result['code'] = str(res.status_code)if dontMangle:- result['content'] = content+ result['content'] = res.contentelse:- result['content'] = unicode(content, 'utf-8', errors='replace')-- #print("FOR: " + url)- #print("HEADERS: " + str(result['headers']))- result['realurl'] = fullPage.geturl()- result['code'] = str(fullPage.getcode())- result['status'] = 'OK'- except urllib2.HTTPError as h:- if not noLog:- self.info("HTTP code " + str(h.code) + " encountered for " + url)- # Capture the HTTP error code- result['code'] = str(h.code)- for k, v in h.info().items():- result['headers'][k.lower()] = v+ result['content'] = unicode(res.content, 'utf-8', errors='replace')if fatal:- self.fatal('URL could not be fetched (' + str(h.code) + ')')- except urllib2.URLError as e:- if not noLog:- self.info("Error fetching " + url + "(" + str(e) + ")")- result['status'] = str(e)- if fatal:- self.fatal('URL could not be fetched (' + str(e) + ')')+ res.raise_for_status()+ except requests.exceptions.HTTPError as h:+ self.fatal('URL could not be fetched (' + str(res.status_code) + ' / ' + res.content + ')')except Exception as x:if not noLog:- self.info("Unexpected exception occurred fetching: " + url + " (" + str(x) + ")")+ try:+ self.error("Unexpected exception (" + str(x) + ") occurred fetching: " + url, False)+ self.error(traceback.format_exc(), False)+ except BaseException as f:+ return resultresult['content'] = Noneresult['status'] = str(x)if fatal:self.fatal('URL could not be fetched (' + str(x) + ')')+ frm = inspect.stack()[1]+ mod = inspect.getmodule(frm[0])+ m = mod.__name__+ atime = time.time()+ t = str(atime - btime)+ self.info("Fetched data: " + str(len(result['content'] or '')) + " (" + url + "), took " + t + "s")return result# Check if wildcard DNS is enabled by looking up two random hostnames@@ -1860,6 +1957,7 @@ class SpiderFootEvent(object):sourceEvent = NonesourceEventHash = NonemoduleDataSource = None+ actualSource = None__id = Nonedef __init__(self, eventType, data, module, sourceEvent,@@ -1923,23 +2021,6 @@ class SpiderFootEvent(object):self.sourceEventHash = srcHash-# Override the default redirectors to re-use cookies-class SmartRedirectHandler(urllib2.HTTPRedirectHandler):- def http_error_301(self, req, fp, code, msg, headers):- if "Set-Cookie" in headers:- req.add_header('cookie', headers['Set-Cookie'])- result = urllib2.HTTPRedirectHandler.http_error_301(- self, req, fp, code, msg, headers)- return result-- def http_error_302(self, req, fp, code, msg, headers):- if "Set-Cookie" in headers:- req.add_header('cookie', headers['Set-Cookie'])- result = urllib2.HTTPRedirectHandler.http_error_302(- self, req, fp, code, msg, headers)- return result--"""Public Suffix List module for Python.See LICENSE.tp for applicable license.