SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
flake8: resolve flake8 violations
Brendan Coles87201e36f0b5
diff --git a/modules/sfp_abusech.py b/modules/sfp_abusech.pyindex 488d0bd6..32f190d9 100644--- a/modules/sfp_abusech.py+++ b/modules/sfp_abusech.py@@ -37,7 +37,7 @@ malchecks = {class sfp_abusech(SpiderFootPlugin):"""abuse.ch:Investigate,Passive:Reputation Systems::Check if a host/domain, IP or netblock is malicious according to abuse.ch."""-+meta = {'name': "abuse.ch",'summary': "Check if a host/domain, IP or netblock is malicious according to abuse.ch.",diff --git a/modules/sfp_abuseipdb.py b/modules/sfp_abuseipdb.pyindex 77a7610a..1e705f83 100644--- a/modules/sfp_abuseipdb.py+++ b/modules/sfp_abuseipdb.py@@ -51,9 +51,9 @@ class sfp_abuseipdb(SpiderFootPlugin):],'favIcon': "https://www.abuseipdb.com/favicon.ico",'logo': "https://www.abuseipdb.com/img/abuseipdb.png.pagespeed.ce.CI8T6WsXU7.png",- 'description': "AbuseIPDB is a project dedicated to helping combat the spread of hackers,"+ 'description': "AbuseIPDB is a project dedicated to helping combat the spread of hackers,""spammers, and abusive activity on the internet.\n"- "Our mission is to help make Web safer by providing a central blacklist for"+ "Our mission is to help make Web safer by providing a central blacklist for""webmasters, system administrators, and other interested parties to""report and find IP addresses that have been associated with malicious activity online."}@@ -130,7 +130,7 @@ class sfp_abuseipdb(SpiderFootPlugin):'Key': self.opts['api_key'],'Accept': "text/plain"}- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],+ data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'], headers=hdr)if data['content'] is None:self.sf.error("Unable to fetch " + url, False)diff --git a/modules/sfp_adblock.py b/modules/sfp_adblock.pyindex 3ddd36a9..f25b844e 100644--- a/modules/sfp_adblock.py+++ b/modules/sfp_adblock.py@@ -35,7 +35,7 @@ class sfp_adblock(SpiderFootPlugin):'logo': "https://adblockplus.org/img/navbar-logo.svg",'description': "Adblock Plus is a free extension that allows you to customize your web experience.""You can block annoying ads, disable tracking and lots more."- "It’s available for all major desktop browsers and for your mobile devices.\n"+ "It’s available for all major desktop browsers and for your mobile devices.\n""Block ads that interrupt your browsing experience.""Say goodbye to video ads, pop-ups, flashing banners and more.""Blocking these annoyances means pages load faster.\n"diff --git a/modules/sfp_alienvault.py b/modules/sfp_alienvault.pyindex 8e58712d..b106ac20 100644--- a/modules/sfp_alienvault.py+++ b/modules/sfp_alienvault.py@@ -18,7 +18,7 @@ from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_alienvault(SpiderFootPlugin):"""AlienVault OTX:Investigate,Passive:Reputation Systems:apikey:Obtain information from AlienVault Open Threat Exchange (OTX)"""-+meta = {'name': "AlienVault OTX",'summary': "Obtain information from AlienVault Open Threat Exchange (OTX)",@@ -100,7 +100,7 @@ class sfp_alienvault(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return ["IP_ADDRESS", "AFFILIATE_IPADDR",+ return ["IP_ADDRESS", "AFFILIATE_IPADDR","NETBLOCK_OWNER", "NETBLOCK_MEMBER"]# What events this module produces@@ -125,7 +125,7 @@ class sfp_alienvault(SpiderFootPlugin):'Accept': 'application/json','X-OTX-API-KEY': self.opts['api_key']}- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot", headers=headers)if res['code'] == "403":diff --git a/modules/sfp_api_recon_dev.py b/modules/sfp_api_recon_dev.pyindex a5bb7519..bd43da93 100644--- a/modules/sfp_api_recon_dev.py+++ b/modules/sfp_api_recon_dev.py@@ -18,7 +18,7 @@ from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_api_recon_dev(SpiderFootPlugin):"""api.recon.dev:Footprint,Investigate,Passive:Passive DNS::Search api.recon.dev for subdomains."""- meta = {+ meta = {'name': "api.recon.dev",'summary': "Search api.recon.dev for subdomains.",'flags': [""],@@ -142,13 +142,13 @@ class sfp_api_recon_dev(SpiderFootPlugin):for domain in set(domains):if self.checkForStop():return None-+if domain in self.results:continueif not self.getTarget().matches(domain, includeChildren=True, includeParents=True):continue-+if self.opts['verify'] and not self.sf.resolveHost(domain):self.sf.debug("Host %s could not be resolved" % domain)evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event)@@ -156,7 +156,7 @@ class sfp_api_recon_dev(SpiderFootPlugin):else:evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event)self.notifyListeners(evt)-+return None# End of sfp_api_recon_dev classdiff --git a/modules/sfp_apility.py b/modules/sfp_apility.pyindex ad5e3092..16b14fa7 100644--- a/modules/sfp_apility.py+++ b/modules/sfp_apility.py@@ -18,7 +18,7 @@ from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_apility(SpiderFootPlugin):"""Apility:Footprint,Investigate,Passive:Reputation Systems:apikey:Search Apility API for IP address and domain reputation."""- meta = {+ meta = {'name': "Apility",'summary': "Search Apility API for IP address and domain reputation.",'flags': ["apikey"],diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.pyindex 0c28c08f..44f077db 100644--- a/modules/sfp_archiveorg.py+++ b/modules/sfp_archiveorg.py@@ -19,7 +19,7 @@ from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_archiveorg(SpiderFootPlugin):"""Archive.org:Footprint,Passive:Search Engines:slow:Identifies historic versions of interesting files/pages from the Wayback Machine."""- meta = {+ meta = {'name': "Archive.org",'summary': "Identifies historic versions of interesting files/pages from the Wayback Machine.",'flags': ["slow"],@@ -44,7 +44,7 @@ class sfp_archiveorg(SpiderFootPlugin):"Today we have 20+ years of web history accessible through the Wayback Machine and we work with 625+ library and ""other partners through our Archive-It program to identify important web pages.",}-+}# Default optionsdiff --git a/modules/sfp_arin.py b/modules/sfp_arin.pyindex 86fc2b5f..8d55cfd6 100644--- a/modules/sfp_arin.py+++ b/modules/sfp_arin.py@@ -18,7 +18,7 @@ from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_arin(SpiderFootPlugin):"""ARIN:Footprint,Investigate,Passive:Public Registries::Queries ARIN registry for contact information."""- meta = {+ meta = {'name': "ARIN",'summary': "Queries ARIN registry for contact information.",'flags': [""],diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.pyindex fdfa0fa5..be3947f3 100644--- a/modules/sfp_azureblobstorage.py+++ b/modules/sfp_azureblobstorage.py@@ -17,7 +17,7 @@ from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_azureblobstorage(SpiderFootPlugin):"""Azure Blob Finder:Footprint,Passive:Crawling and Scanning::Search for potential Azure blobs associated with the target and attempt to list their contents."""-+meta = {'name': "Azure Blob Finder",'summary': "Search for potential Azure blobs associated with the target and attempt to list their contents.",diff --git a/modules/sfp_bambenek.py b/modules/sfp_bambenek.pyindex 58197716..2faf493d 100644--- a/modules/sfp_bambenek.py+++ b/modules/sfp_bambenek.py@@ -54,7 +54,7 @@ class sfp_bambenek(SpiderFootPlugin):"Every day, there is another story about another company having their banking accounts drained, ""someone having their identity stolen, or critical infrastructure being taken offline by hostile entities. ""Led by IT security expert, John Bambenek, we have the resources to bring to your business so "- "you can be sure your organization and your customers’ data is safe.",+ "you can be sure your organization and your customers’ data is safe.",}}diff --git a/modules/sfp_bgpview.py b/modules/sfp_bgpview.pyindex be8b5d1c..849701a2 100644--- a/modules/sfp_bgpview.py+++ b/modules/sfp_bgpview.py@@ -35,7 +35,7 @@ class sfp_bgpview(SpiderFootPlugin):'description': "BGPView is a simple API allowing consumers to view all sort of analytics data about the current state and structure of the internet.",}}-+opts = {}diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.pyindex 077cacee..cf55d344 100644--- a/modules/sfp_bingsharedip.py+++ b/modules/sfp_bingsharedip.py@@ -153,13 +153,11 @@ class sfp_bingsharedip(SpiderFootPlugin):if not self.opts["cohostsamedomain"]:if self.getTarget().matches(site, includeParents=True):self.sf.debug(- "Skipping "- + site- + " because it is on the same domain."+ f"Skipping {site} because it is on the same domain.")continueif self.opts["verify"] and not self.sf.validateIP(site, ip):- self.sf.debug("Host " + site + " no longer resolves to " + ip)+ self.sf.debug(f"Host {site} no longer resolves to {ip}")continue# Create an IP Address event stemming from the netblock as the# link to the co-host.diff --git a/modules/sfp_builtwith.py b/modules/sfp_builtwith.pyindex 629af873..e807a658 100644--- a/modules/sfp_builtwith.py+++ b/modules/sfp_builtwith.py@@ -85,7 +85,7 @@ class sfp_builtwith(SpiderFootPlugin):# What events this module producesdef producedEvents(self):return ["INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",- "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME",+ "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME","CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID"]def queryRelationships(self, t):@@ -244,7 +244,7 @@ class sfp_builtwith(SpiderFootPlugin):evttype = "INTERNET_NAME"else:evttype = "CO_HOSTED_SITE"-+# Create the name/co-hoste = SpiderFootEvent(evttype, val, self.__name__, event)self.notifyListeners(e)diff --git a/modules/sfp_censys.py b/modules/sfp_censys.pyindex a894f6b3..324703df 100644--- a/modules/sfp_censys.py+++ b/modules/sfp_censys.py@@ -86,7 +86,7 @@ class sfp_censys(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ["BGP_AS_MEMBER", "TCP_PORT_OPEN", "OPERATING_SYSTEM",+ return ["BGP_AS_MEMBER", "TCP_PORT_OPEN", "OPERATING_SYSTEM","WEBSERVER_HTTPHEADERS", "NETBLOCK_MEMBER", "GEOINFO",'RAW_RIR_DATA']@@ -97,7 +97,7 @@ class sfp_censys(SpiderFootPlugin):querytype = "ipv4/{0}"if querytype == "host":querytype = "websites/{0}"-+secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret']b64_val = base64.b64encode(secret.encode('utf-8'))@@ -242,5 +242,5 @@ class sfp_censys(SpiderFootPlugin):self.notifyListeners(e)except BaseException as e:self.sf.error("Error encountered processing record for " + eventData + " (" + str(e) + ")", False)-+# End of sfp_censys classdiff --git a/modules/sfp_countryname.py b/modules/sfp_countryname.pyindex d161c383..620a2dab 100644--- a/modules/sfp_countryname.py+++ b/modules/sfp_countryname.py@@ -339,7 +339,7 @@ class sfp_countryname(SpiderFootPlugin):return nonCountryCodesDict- # Detect name of country from phone number+ # Detect name of country from phone numberdef detectCountryFromPhone(self, srcPhoneNumber):# Get dictionary of country codes and country names@@ -358,18 +358,18 @@ class sfp_countryname(SpiderFootPlugin):self.sf.debug("Skipped invalid phone number: " + srcPhoneNumber)return None-+# Detect name of country from TLD of domain namedef detectCountryFromTLD(self, srcDomain):-+# Get dictionary of TLD country codes and country namestldCountryCodes = self.getCountryCodeDict()-+# Get dictionary of non country TLD codestldNonCountryCodes = self.getNonCountryCodesDict()- # Split domain into parts by '.'- # Country TLDs are reserved+ # Split domain into parts by '.'+ # Country TLDs are reserveddomainParts = srcDomain.split(".")# Search for country TLD in the domain parts - reversed@@ -391,22 +391,22 @@ class sfp_countryname(SpiderFootPlugin):# Get dictionary of country codes and country namestldCountryCodes = self.getCountryCodeDict()try:- # Get country code from IBAN+ # Get country code from IBANcountryCode = srcIBAN[0:2].upper()return tldCountryCodes[countryCode]except:# No country name is found in the IBANreturn None-+# Detect name of country from whois lookup, Geo Info, Physical Address datadef detectCountryFromData(self, srcData):-+# Get dictionary of country codes and country namesabbvCountryCodes = self.getCountryCodeDict()countries = list()# Look for countrycodes and country in source data- for countryName in abbvCountryCodes.values():+ for countryName in abbvCountryCodes.values():# Look for country name in source data# Spaces are not included since New Jersey and others@@ -440,7 +440,7 @@ class sfp_countryname(SpiderFootPlugin):return countriesreturn None-+# What events is this module interested in for inputdef watchedEvents(self):return ["IBAN_NUMBER", "PHONE_NUMBER", "AFFILIATE_DOMAIN_NAME",@@ -457,7 +457,7 @@ class sfp_countryname(SpiderFootPlugin):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data-+evttype = "COUNTRY_NAME"self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")@@ -467,8 +467,8 @@ class sfp_countryname(SpiderFootPlugin):# Do not parse duplicate incoming dataif eventDataHash in self.results:self.sf.debug("Already found from this source")- return None-+ return None+self.results[eventDataHash] = TruecountryNames = list()@@ -486,7 +486,7 @@ class sfp_countryname(SpiderFootPlugin):countryNames.append(None)else:countryNames.extend(tempDataList)-+# Check if countryNames is emptyif len(countryNames) == 0:return None@@ -506,5 +506,5 @@ class sfp_countryname(SpiderFootPlugin):evt.moduleDataSource = "Unknown"self.notifyListeners(evt)return None-+# End of sfp_countryname classdiff --git a/modules/sfp_h1nobbdde.py b/modules/sfp_h1nobbdde.pyindex 3a7ba092..8508d7cc 100644--- a/modules/sfp_h1nobbdde.py+++ b/modules/sfp_h1nobbdde.py@@ -76,14 +76,15 @@ class sfp_h1nobbdde(SpiderFootPlugin):return Nonetry:- rx = re.compile("<a class=\"title\" href=.(.[^\"]+).*?title=.(.[^\"\']+)", re.IGNORECASE|re.DOTALL)+ rx = re.compile("<a class=\"title\" href=.(.[^\"]+).*?title=.(.[^\"\']+)", re.IGNORECASE | re.DOTALL)for m in rx.findall(res['content']):# Report itif qry in m[1]:ret.append(m[1] + "\n<SFURL>" + m[0] + "</SFURL>")except Exception as e:- self.sf.error("Error processing response from h1.nobbd.de: " + str(e), False)+ self.sf.error(f"Error processing response from h1.nobbd.de: {e}", False)return None+return retdef handleEvent(self, event):@@ -97,8 +98,8 @@ class sfp_h1nobbdde(SpiderFootPlugin):if eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True++ self.results[eventData] = Trueobb = self.queryOBB(eventData)if obb:diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex 3a48b756..30b03881 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -88,36 +88,41 @@ class sfp_spider(SpiderFootPlugin):cookies = None# Filter out certain file types (if user chooses to)- checkExts = lambda ext: url.lower().split('?')[0].endswith('.' + ext.lower())- if list(filter(checkExts, self.opts['filterfiles'])):- #self.sf.debug('Ignoring filtered extension: ' + link)+ if list(filter(lambda ext: url.lower().split('?')[0].endswith('.' + ext.lower()), self.opts['filterfiles'])):+ # self.sf.debug('Ignoring filtered extension: ' + link)return Noneif site in self.siteCookies:- self.sf.debug("Restoring cookies for " + site + ": " + str(self.siteCookies[site]))+ self.sf.debug(f"Restoring cookies for {site}: {self.siteCookies[site]}")cookies = self.siteCookies[site]# Fetch the contents of the supplied URL (object returned)- fetched = self.sf.fetchUrl(url, False, cookies,- self.opts['_fetchtimeout'], self.opts['_useragent'],- sizeLimit=10000000,- verify=False)+ fetched = self.sf.fetchUrl(+ url,+ False,+ cookies,+ self.opts['_fetchtimeout'],+ self.opts['_useragent'],+ sizeLimit=10000000,+ verify=False+ )self.fetchedPages[url] = True# Track cookies a site has sent, then send the back in subsquent requestsif self.opts['usecookies'] and fetched['headers'] is not None:if fetched['headers'].get('Set-Cookie'):self.siteCookies[site] = fetched['headers'].get('Set-Cookie')- self.sf.debug("Saving cookies for " + site + ": " + str(self.siteCookies[site]))+ self.sf.debug(f"Saving cookies for {site}: {self.siteCookies[site]}")if url not in self.urlEvents:- self.sf.error("Something strange happened - shouldn't get here.", False)+ # TODO: be more descriptive+ self.sf.error("Something strange happened - shouldn't get here: url not in self.urlEvents", False)self.urlEvents[url] = None# Notify modules about the content obtainedself.contentNotify(url, fetched, self.urlEvents[url])if fetched['realurl'] is not None and fetched['realurl'] != url:- #self.sf.debug("Redirect of " + url + " to " + fetched['realurl'])+ # self.sf.debug("Redirect of " + url + " to " + fetched['realurl'])# Store the content for the redirect so that it isn't fetched againself.fetchedPages[fetched['realurl']] = True# Notify modules about the new link@@ -129,8 +134,8 @@ class sfp_spider(SpiderFootPlugin):links = self.sf.parseLinks(url, fetched['content'],self.getTarget().getNames())- if links is None or len(links) == 0:- self.sf.info("No links found at " + url)+ if not links:+ self.sf.info(f"No links found at {url}")return None# Notify modules about the links found@@ -156,30 +161,29 @@ class sfp_spider(SpiderFootPlugin):# Skip external sites (typical behaviour..)if not self.getTarget().matches(linkFQDN):- #self.sf.debug('Ignoring external site: ' + link)+ # self.sf.debug('Ignoring external site: ' + link)continue# Optionally skip sub-domain sitesif self.opts['nosubs'] and not \self.getTarget().matches(linkFQDN, includeChildren=False):- #self.sf.debug("Ignoring subdomain: " + link)+ # self.sf.debug("Ignoring subdomain: " + link)continue# Skip parent domain sitesif not self.getTarget().matches(linkFQDN, includeParents=False):- #self.sf.debug("Ignoring parent domain: " + link)+ # self.sf.debug("Ignoring parent domain: " + link)continue# Optionally skip user directoriesif self.opts['filterusers'] and '/~' in link:- #self.sf.debug("Ignoring user folder: " + link)+ # self.sf.debug("Ignoring user folder: " + link)continue# If we are respecting robots.txt, filter those out tooif linkBase in self.robotsRules and self.opts['robotsonly']:- checkRobots = lambda blocked: type(blocked).lower(blocked) in link.lower() or blocked == '*'- if list(filter(checkRobots, self.robotsRules[linkBase])):- #self.sf.debug("Ignoring page found in robots.txt: " + link)+ if list(filter(lambda blocked: type(blocked).lower(blocked) in link.lower() or blocked == '*', self.robotsRules[linkBase])):+ # self.sf.debug("Ignoring page found in robots.txt: " + link)continue# All tests passed, add link to be spidered@@ -363,14 +367,13 @@ class sfp_spider(SpiderFootPlugin):breaknextLinks = self.cleanLinks(links)- self.sf.info("Found links: " + str(nextLinks))+ self.sf.info(f"Found links: {nextLinks}")# We've scanned through another layer of the sitelevelsTraversed += 1- self.sf.debug("At level: " + str(levelsTraversed) + ", Pages: " + str(totalFetched))+ self.sf.debug(f"At level: {levelsTraversed}, Pages: {totalFetched}")if levelsTraversed >= self.opts['maxlevels']:- self.sf.info("Maximum number of levels (" + str(self.opts['maxlevels']) +- ") reached.")+ self.sf.info(f"Maximum number of levels ({self.opts['maxlevels']}) reached.")keepSpidering = False# We've reached the end of our journey..diff --git a/modules/sfp_talosintel.py b/modules/sfp_talosintel.pyindex 68f92d79..654b3ff3 100644--- a/modules/sfp_talosintel.py+++ b/modules/sfp_talosintel.py@@ -144,11 +144,10 @@ class sfp_talosintel(SpiderFootPlugin):try:if IPAddress(ip) in IPNetwork(target):- self.sf.debug(ip + " found within netblock/subnet "- + target + " in " + check)+ self.sf.debug(f"{ip} found within netblock/subnet {target} in {check}")return urlexcept Exception as e:- self.sf.debug("Error encountered parsing: " + str(e))+ self.sf.debug(f"Error encountered parsing: {e}")continuereturn None@@ -167,7 +166,7 @@ class sfp_talosintel(SpiderFootPlugin):for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")+ self.sf.debug(f"{target}/{targetDom} found in {check} list.")return urlexcept BaseException as e:self.sf.debug("Error encountered parsing 2: " + str(e))@@ -179,7 +178,7 @@ class sfp_talosintel(SpiderFootPlugin):for check in list(malchecks.keys()):cid = malchecks[check]['id']if cid == resourceId and itemType in malchecks[check]['checks']:- self.sf.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid)+ self.sf.debug(f"Checking maliciousness of {target} ({itemType}) with: {cid}")return self.resourceList(cid, target, itemType)return Nonediff --git a/setup.cfg b/setup.cfgindex b5bce099..27867363 100644--- a/setup.cfg+++ b/setup.cfg@@ -1,3 +1,3 @@[flake8]flake8-max-line-length = 120-ignore = F841 E501 E131 E401 E201 E302 E265 W605 W504 E121 E226 E722 E127 E741 E126 E711 E303 E502 E123 E301 E125 E262 E227 E231 E712 W391 W503 E731 W291 W293+ignore = F841 E501 W503 E131 E401 E201 E302 E265 W605 W504 E121 E226 E722 E127 E741 E126 E711 E303 E502 E123 E301 E125 E262 E712