SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
flake8: resolve flake8 violations
Brendan Colescc3048465e55
diff --git a/modules/sfp_badpackets.py b/modules/sfp_badpackets.pyindex a42b9e26..bf25aa57 100644--- a/modules/sfp_badpackets.py+++ b/modules/sfp_badpackets.py@@ -252,13 +252,14 @@ class sfp_badpackets(SpiderFootPlugin):category = record.get('tags')[0].get('category')if category:maliciousIPDesc += " - CATEGORY : " + str(category) + "\n"- except:+ except BaseException:self.sf.debug("No category found for target")+try:description = record.get('tags')[0].get('description')if description:maliciousIPDesc += " - DESCRIPTION : " + str(description) + "\n"- except:+ except BaseException:self.sf.debug("No description found for target")maliciousIPDescHash = self.sf.hashstring(maliciousIPDesc)diff --git a/modules/sfp_countryname.py b/modules/sfp_countryname.pyindex 63dc1b2a..e1fd7e5c 100644--- a/modules/sfp_countryname.py+++ b/modules/sfp_countryname.py@@ -70,13 +70,13 @@ class sfp_countryname(SpiderFootPlugin):try:phoneNumber = phonenumbers.parse(srcPhoneNumber)- except:+ except BaseException:self.sf.debug(f"Skipped invalid phone number: {srcPhoneNumber}")return Nonetry:countryCode = region_code_for_country_code(phoneNumber.country_code)- except:+ except BaseException:self.sf.debug(f"Lookup of region code failed for phone number: {srcPhoneNumber}")return Nonediff --git a/modules/sfp_dnsgrep.py b/modules/sfp_dnsgrep.pyindex fcf16ed9..8e9e7a76 100644--- a/modules/sfp_dnsgrep.py+++ b/modules/sfp_dnsgrep.py@@ -136,7 +136,7 @@ class sfp_dnsgrep(SpiderFootPlugin):for r in fdns:try:ip, domain = r.split(',')- except:+ except BaseException:continuedomains.append(domain)@@ -147,7 +147,7 @@ class sfp_dnsgrep(SpiderFootPlugin):for r in rdns:try:ip, domain = r.split(',')- except:+ except BaseException:continuedomains.append(domain)diff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.pyindex d5dcd33c..05362403 100644--- a/modules/sfp_dnszonexfer.py+++ b/modules/sfp_dnszonexfer.py@@ -28,11 +28,9 @@ class sfp_dnszonexfer(SpiderFootPlugin):'categories': ["DNS"]}- # Default optionsopts = {}- # Option descriptionsoptdescs = {}@@ -46,17 +44,12 @@ class sfp_dnszonexfer(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ['PROVIDER_DNS']- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):return ["RAW_DNS_RECORDS", "INTERNET_NAME"]- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -67,7 +60,7 @@ class sfp_dnszonexfer(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")if srcModuleName == "sfp_dnszonexfer":- self.sf.debug("Ignoring " + eventName + ", from self.")+ self.sf.debug(f"Ignoring {eventName}, from self.")return Noneif eventDataHash in self.events:@@ -94,8 +87,7 @@ class sfp_dnszonexfer(SpiderFootPlugin):nsip = nbreakelse:- self.sf.error("Couldn't resolve the name server, " + \- "so not attempting zone transfer.", False)+ self.sf.error("Couldn't resolve the name server, so not attempting zone transfer.", False)return Noneelse:nsip = eventData@@ -124,12 +116,10 @@ class sfp_dnszonexfer(SpiderFootPlugin):else:strdata = strdata + "." + name- evt = SpiderFootEvent("INTERNET_NAME", strdata,- self.__name__, parentEvent)+ evt = SpiderFootEvent("INTERNET_NAME", strdata, self.__name__, parentEvent)self.notifyListeners(evt)except BaseException as e:- self.sf.info("Unable to perform DNS zone transfer for " + eventData- + "(" + name + "): " + str(e))+ self.sf.info(f"Unable to perform DNS zone transfer for {eventData} ({name}): {e}")# End of sfp_dnszonexfer classdiff --git a/modules/sfp_dronebl.py b/modules/sfp_dronebl.pyindex df1815a8..9ad4d66d 100644--- a/modules/sfp_dronebl.py+++ b/modules/sfp_dronebl.py@@ -162,27 +162,28 @@ class sfp_dronebl(SpiderFootPlugin):if eventData in self.results:return None+self.results[eventData] = Trueif eventName == 'NETBLOCK_OWNER':if not self.opts['netblocklookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:- self.sf.debug("Network size bigger than permitted: "- + str(IPNetwork(eventData).prefixlen) + " > "- + str(self.opts['maxnetblock']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:+ self.sf.debug("Network size bigger than permitted: "+ + str(IPNetwork(eventData).prefixlen) + " > "+ + str(self.opts['maxnetblock']))+ return Noneif eventName == 'NETBLOCK_MEMBER':if not self.opts['subnetlookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:- self.sf.debug("Network size bigger than permitted: "- + str(IPNetwork(eventData).prefixlen) + " > "- + str(self.opts['maxsubnet']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:+ self.sf.debug("Network size bigger than permitted: "+ + str(IPNetwork(eventData).prefixlen) + " > "+ + str(self.opts['maxsubnet']))+ return Noneif eventName.startswith("NETBLOCK_"):for addr in IPNetwork(eventData):diff --git a/modules/sfp_errors.py b/modules/sfp_errors.pyindex fde9cc51..c8cdf648 100644--- a/modules/sfp_errors.py+++ b/modules/sfp_errors.py@@ -106,9 +106,4 @@ class sfp_errors(SpiderFootPlugin):return None- # If you intend for this module to act on its own (e.g. not solely rely- # on events from other modules, then you need to have a start() method- # and within that method call self.checkForStop() to see if you've been- # politely asked by the controller to stop your activities (user abort.)-# End of sfp_errors classdiff --git a/modules/sfp_fsecure_riddler.py b/modules/sfp_fsecure_riddler.pyindex 954368f1..43a26a1a 100644--- a/modules/sfp_fsecure_riddler.py+++ b/modules/sfp_fsecure_riddler.py@@ -107,7 +107,7 @@ class sfp_fsecure_riddler(SpiderFootPlugin):try:token = data.get('response').get('user').get('authentication_token')- except:+ except BaseException:self.sf.error('Login failed', False)self.errorState = Truereturn Nonediff --git a/modules/sfp_fullcontact.py b/modules/sfp_fullcontact.pyindex 982559bb..f4ebe460 100644--- a/modules/sfp_fullcontact.py+++ b/modules/sfp_fullcontact.py@@ -148,7 +148,6 @@ class sfp_fullcontact(SpiderFootPlugin):return self.query(url, q)- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -157,30 +156,36 @@ class sfp_fullcontact(SpiderFootPlugin):if self.errorState:return None- if self.opts['api_key'] == "":- self.sf.error("You enabled sfp_fullcontact but did not set an API key!", False)+ if self.opts["api_key"] == "":+ self.sf.error(+ f"You enabled {self.__class__.__name__} but did not set an API key!",+ False,+ )self.errorState = Truereturn Noneself.sf.debug(f"Received event, {eventName}, from {srcModuleName}")- # Don't look up stuff twiceif eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True++ self.results[eventData] = Trueif eventName == "EMAILADDR":data = self.queryPerson(email=eventData)+if not data:return None- if not data.get('fullName'):++ full_name = data.get('fullName')++ if not full_name:return None- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + \- data['fullName'], self.__name__, event)++ e = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event)self.notifyListeners(e)- return+ return Noneif eventName == "DOMAIN_NAME":data = self.queryCompany(eventData)@@ -219,9 +224,12 @@ class sfp_fullcontact(SpiderFootPlugin):if data.get("keyPeople"):for r in data['keyPeople']:if r.get('fullName'):- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + \- r['fullName'], self.__name__, event)+ e = SpiderFootEvent(+ "RAW_RIR_DATA",+ "Possible full name: {r['fullName']}",+ self.__name__,+ event+ )self.notifyListeners(e)-# End of sfp_fullcontact classdiff --git a/modules/sfp_googlemaps.py b/modules/sfp_googlemaps.pyindex fa22502b..addae551 100644--- a/modules/sfp_googlemaps.py+++ b/modules/sfp_googlemaps.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import urllibimport jsonfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin@@ -49,7 +50,6 @@ class sfp_googlemaps(SpiderFootPlugin):}}- # Default optionsopts = {"api_key": ""}@@ -67,17 +67,30 @@ class sfp_googlemaps(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ['DOMAIN_NAME', 'PHYSICAL_ADDRESS']- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):return ["PHYSICAL_ADDRESS", "PHYSICAL_COORDINATES", "RAW_RIR_DATA"]- # Handle events sent to this module+ def query(self, address):+ params = urllib.parse.urlencode({+ 'key': self.opts['api_key'],+ 'address': address.encode('raw_unicode_escape').decode("ascii", errors='replace')+ })++ res = self.sf.fetchUrl(+ "https://maps.googleapis.com/maps/api/geocode/json?{params}",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent']+ )++ if res['content'] is None:+ self.sf.info(f"No location info found for {address}")+ return None++ return res+def handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -86,51 +99,62 @@ class sfp_googlemaps(SpiderFootPlugin):if self.errorState:return None+ if eventData in self.results:+ self.sf.debug(f"Skipping {eventData}, already checked.")+ return None++ self.results[eventData] = True+self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")- if self.opts['api_key'] == "":- self.sf.error("You enabled sfp_googlemaps but did not set an API key!", False)+ if self.opts["api_key"] == "":+ self.sf.error(+ f"You enabled {self.__class__.__name__} but did not set an API key!",+ False,+ )self.errorState = Truereturn None- # Don't look up stuff twice- if eventData in self.results:- self.sf.debug(f"Skipping {eventData}, already checked.")- return None- else:- self.results[eventData] = True+ res = self.query(eventData)- res = self.sf.fetchUrl("https://maps.googleapis.com/maps/api/geocode/json?address=" + \- eventData + "&key=" + self.opts['api_key'],- timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])- if res['content'] is None:- self.sf.info("No location info found for " + eventData)+ if not res:+ self.sf.debug(f"No information found for {eventData}")return None- evt = SpiderFootEvent("RAW_RIR_DATA", res['content'],- self.__name__, event)+ evt = SpiderFootEvent(+ "RAW_RIR_DATA",+ res['content'],+ self.__name__,+ event+ )self.notifyListeners(evt)try:data = json.loads(res['content'])['results'][0]-- if eventName in ["PHYSICAL_ADDRESS", "DOMAIN_NAME"] and \- srcModuleName != "sfp_googlemaps":- if 'geometry' in data:- lat = str(data['geometry']['location']['lat'])- lng = str(data['geometry']['location']['lng'])- evt = SpiderFootEvent("PHYSICAL_COORDINATES", lat + "," + lng, self.__name__, event)- self.notifyListeners(evt)-- if 'formatted_address' in data:- evt = SpiderFootEvent("PHYSICAL_ADDRESS", data['formatted_address'],- self.__name__, event)- self.notifyListeners(evt)except Exception as e:- self.sf.debug("Error processing JSON response: " + str(e))+ self.sf.debug(f"Error processing JSON response: {e}")return None+ if srcModuleName != "sfp_googlemaps":+ geometry = data.get('geometry')+ if geometry:+ location = data.get('location')+ if location:+ lat = location.get('lat')+ lng = location.get('lng')+ if lat and lng:+ evt = SpiderFootEvent("PHYSICAL_COORDINATES", f"{lat},{lng}", self.__name__, event)+ self.notifyListeners(evt)++ if 'formatted_address' in data:+ evt = SpiderFootEvent(+ "PHYSICAL_ADDRESS",+ data['formatted_address'],+ self.__name__,+ event+ )+ self.notifyListeners(evt)+return None# End of sfp_googlemaps classdiff --git a/modules/sfp_instagram.py b/modules/sfp_instagram.pyindex 75ad3514..ad40452a 100644--- a/modules/sfp_instagram.py+++ b/modules/sfp_instagram.py@@ -36,11 +36,9 @@ class sfp_instagram(SpiderFootPlugin):}}- # Default optionsopts = {}- # Option descriptionsoptdescs = {}@@ -53,15 +51,12 @@ class sfp_instagram(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ['SOCIAL_MEDIA']- # What events this module producesdef producedEvents(self):return ['RAW_RIR_DATA']- # Extract profile JSON from HTMLdef extractJson(self, html):m = r'<script type="application/ld\+json">(.+?)</script>'json_data = re.findall(m, html, re.MULTILINE | re.DOTALL)@@ -77,7 +72,6 @@ class sfp_instagram(SpiderFootPlugin):return data- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -95,19 +89,19 @@ class sfp_instagram(SpiderFootPlugin):network = eventData.split(": ")[0]url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:- self.sf.error("Unable to parse SOCIAL_MEDIA: "- + eventData + " (" + str(e) + ")", False)+ self.sf.error(f"Unable to parse SOCIAL_MEDIA: {eventData} ({e})", False)return Noneif not network == 'Instagram':- self.sf.debug("Skipping social network profile, " + url + \- ", as not an Instagram profile")+ self.sf.debug(f"Skipping social network profile, {url}, as not an Instagram profile")return None# Retrieve profile- res = self.sf.fetchUrl(url,- timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ url,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent']+ )if res['content'] is None:self.sf.debug('No response from Instagram.com')@@ -117,7 +111,7 @@ class sfp_instagram(SpiderFootPlugin):json_data = self.extractJson(res['content'])if not json_data:- self.sf.debug(url + " is not a valid Instagram profile")+ self.sf.debug(f"{url} is not a valid Instagram profile")return Nonee = SpiderFootEvent('RAW_RIR_DATA', str(json_data), self.__name__, event)diff --git a/modules/sfp_keybase.py b/modules/sfp_keybase.pyindex 077fe790..eae0bb41 100644--- a/modules/sfp_keybase.py+++ b/modules/sfp_keybase.py@@ -106,7 +106,7 @@ class sfp_keybase(SpiderFootPlugin):try:if not int(code) == 0:return None- except:+ except BaseException:self.sf.error("Invalid code returned as response", False)return None@@ -165,7 +165,7 @@ class sfp_keybase(SpiderFootPlugin):# Contains all data about the target usernametry:them = content.get('them')[0]- except:+ except BaseException:self.sf.error("Invalid data received", False)them = Nonediff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.pyindex 44c2a115..0439a7ef 100644--- a/modules/sfp_myspace.py+++ b/modules/sfp_myspace.py@@ -38,11 +38,9 @@ class sfp_myspace(SpiderFootPlugin):}}- # Default optionsopts = {}- # Option descriptionsoptdescs = {}@@ -56,15 +54,12 @@ class sfp_myspace(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ["EMAILADDR", "SOCIAL_MEDIA"]- # What events this module producesdef producedEvents(self):return ["SOCIAL_MEDIA", "GEOINFO"]- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -72,8 +67,8 @@ class sfp_myspace(SpiderFootPlugin):if eventData in self.results:return None- else:- self.results[eventData] = True++ self.results[eventData] = Trueself.sf.debug(f"Received event, {eventName}, from {srcModuleName}")@@ -85,14 +80,14 @@ class sfp_myspace(SpiderFootPlugin):useragent=self.opts['_useragent'])if res['content'] is None:- self.sf.error("Could not fetch MySpace content for " + email, False)+ self.sf.error(f"Could not fetch MySpace content for {email}", False)return None# Extract HTML containing potential profile matchesprofiles = re.findall(r'<a href="/[a-zA-Z0-9_]+">[^<]+</a></h6>', res['content'])if not profiles:- self.sf.debug("No profiles found with that e-mail.")+ self.sf.debug(f"No profiles found for e-mail: {email}")return None# The first result is the closest match, but whether it's an exact match is unknown.@@ -110,9 +105,12 @@ class sfp_myspace(SpiderFootPlugin):return Nonename = matches[0]- e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + \- "<SFURL>https://myspace.com/" + name + "</SFURL>",- self.__name__, event)+ e = SpiderFootEvent(+ "SOCIAL_MEDIA",+ f"MySpace: <SFURL>https://myspace.com/{name}</SFURL>",+ self.__name__,+ event+ )self.notifyListeners(e)# Retrieve location from MySpace profile@@ -126,8 +124,7 @@ class sfp_myspace(SpiderFootPlugin):return Noneif network != "MySpace":- self.sf.debug("Skipping social network profile, " + url + \- ", as not a MySpace profile")+ self.sf.debug(f"Skipping social network profile, {url}, as not a MySpace profile")return Noneres = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.pyindex a1c3f68d..5657b077 100644--- a/modules/sfp_opencorporates.py+++ b/modules/sfp_opencorporates.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import urllibimport jsonfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin@@ -39,13 +40,11 @@ class sfp_opencorporates(SpiderFootPlugin):}}- # Default optionsopts = {'confidence': 100,'api_key': ''}- # Option descriptionsoptdescs = {'confidence': "Confidence that the search result objects are correct (numeric value between 0 and 100).",'api_key': 'OpenCorporates.com API key.'@@ -60,26 +59,40 @@ class sfp_opencorporates(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ["COMPANY_NAME"]- # What events this module producesdef producedEvents(self):return ["COMPANY_NAME", "PHYSICAL_ADDRESS", "RAW_RIR_DATA"]- # Search for company name- # https://api.opencorporates.com/documentation/API-Referencedef searchCompany(self, qry):+ """Search for company name++ Args:+ qry (str): company name++ Returns:+ str+ """++ version = '0.4'+apiparam = ""if not self.opts['api_key'] == "":apiparam = "&api_token=" + self.opts['api_key']- # High timeouts as they can sometimes take a while- res = self.sf.fetchUrl("https://api.opencorporates.com/v0.4/companies/search?q=" + \- qry + "&format=json&order=score&confidence=" + \- str(self.opts['confidence']) + apiparam,- timeout=60, useragent=self.opts['_useragent'])+ params = urllib.parse.urlencode({+ 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),+ 'format': 'json',+ 'order': 'score',+ 'confidence': self.opts['confidence']+ })++ res = self.sf.fetchUrl(+ "https://api.opencorporates.com/v{version}/companies/search?{params}{apiparam}",+ timeout=60, # High timeouts as they can sometimes take a while+ useragent=self.opts['_useragent']+ )if res['code'] == "401":self.sf.error("Invalid OpenCorporates API key.", False)@@ -89,7 +102,6 @@ class sfp_opencorporates(SpiderFootPlugin):self.sf.error("You are being rate-limited by OpenCorporates.", False)return None- # Parse response content as JSONtry:data = json.loads(res['content'])except Exception as e:@@ -101,15 +113,17 @@ class sfp_opencorporates(SpiderFootPlugin):return data['results']- # Retrieve company details- # https://api.opencorporates.com/documentation/API-Referencedef retrieveCompanyDetails(self, jurisdiction_code, company_number):- url = "https://api.opencorporates.com/companies/" + jurisdiction_code + "/" + str(company_number)+ url = f"https://api.opencorporates.com/companies/{jurisdiction_code}/{company_number}"if not self.opts['api_key'] == "":- url += '?' + self.opts['api_key']+ url += "?api_token=" + self.opts['api_key']- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ url,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent']+ )if res['code'] == "401":self.sf.error("Invalid OpenCorporates API key.", False)@@ -119,7 +133,6 @@ class sfp_opencorporates(SpiderFootPlugin):self.sf.error("You are being rate-limited by OpenCorporates.", False)return None- # Parse response content as JSONtry:data = json.loads(res['content'])except Exception as e:@@ -174,7 +187,6 @@ class sfp_opencorporates(SpiderFootPlugin):e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + n, self.__name__, sevt)self.notifyListeners(e)- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -183,13 +195,12 @@ class sfp_opencorporates(SpiderFootPlugin):if eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True++ self.results[eventData] = Trueself.sf.debug(f"Received event, {eventName}, from {srcModuleName}")- # Search for the company- res = self.searchCompany(eventData + "*")+ res = self.searchCompany(f"{eventData}*")if res is None:self.sf.debug("Found no results for " + eventData)diff --git a/modules/sfp_phishstats.py b/modules/sfp_phishstats.pyindex 18030caa..f2d4cc73 100644--- a/modules/sfp_phishstats.py+++ b/modules/sfp_phishstats.py@@ -112,8 +112,8 @@ class sfp_phishstats(SpiderFootPlugin):try:return json.loads(res['content'])- except:- self.sf.error("Ill formatted data received as JSON response", False)+ except BaseException as e:+ self.sf.error(f"Error processing JSON response: {e}", False)return None# Handle events sent to this module@@ -177,7 +177,7 @@ class sfp_phishstats(SpiderFootPlugin):try:maliciousIP = data[0].get('ip')- except:+ except BaseException:# If ArrayIndex is out of bounds then data doesn't existcontinuediff --git a/modules/sfp_psbdmp.py b/modules/sfp_psbdmp.pyindex 16e8860d..fa7015c5 100644--- a/modules/sfp_psbdmp.py+++ b/modules/sfp_psbdmp.py@@ -38,40 +38,26 @@ class sfp_psbdmp(SpiderFootPlugin):}}- # Default optionsopts = {}- # Option descriptionsoptdescs = {}- # Be sure to completely clear any class variables in setup()- # or you run the risk of data persisting between scan runs.-results = Nonedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()- # Clear / reset any other class member variables here- # or you risk them persisting between threads.-for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):- ret = ["EMAILADDR", "DOMAIN_NAME", "INTERNET_NAME"]-- return ret+ return ["EMAILADDR", "DOMAIN_NAME", "INTERNET_NAME"]- # What events this module producesdef producedEvents(self):- ret = ["LEAKSITE_URL", "LEAKSITE_CONTENT"]-- return ret+ return ["LEAKSITE_URL", "LEAKSITE_CONTENT"]def query(self, qry):ret = None@@ -105,7 +91,6 @@ class sfp_psbdmp(SpiderFootPlugin):return ids- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -113,12 +98,11 @@ class sfp_psbdmp(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")- # Don't look up stuff twiceif eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True++ self.results[eventData] = Truedata = self.query(eventData)if data is None:@@ -128,22 +112,28 @@ class sfp_psbdmp(SpiderFootPlugin):e = SpiderFootEvent("LEAKSITE_URL", n, self.__name__, event)self.notifyListeners(e)- res = self.sf.fetchUrl(n, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ n,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent']+ )if res['content'] is None:- self.sf.debug("Ignoring " + n + " as no data returned")+ self.sf.debug(f"Ignoring {n} as no data returned")continue# Sometimes pastes search results false positives- if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData)- + r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ if re.search(+ r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) + r"[^a-zA-Z\-\_0-9]",+ res['content'],+ re.IGNORECASE+ ) is None:continuetry:startIndex = res['content'].index(eventData)except BaseException:- self.sf.debug("String not found in pastes content.")+ self.sf.debug(f"{eventData} not found in pastes content.")continueevt = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, e)diff --git a/modules/sfp_quad9.py b/modules/sfp_quad9.pyindex 2e989112..c1f3a169 100644--- a/modules/sfp_quad9.py+++ b/modules/sfp_quad9.py@@ -42,11 +42,9 @@ class sfp_quad9(SpiderFootPlugin):}}- # Default optionsopts = {}- # Option descriptionsoptdescs = {}@@ -59,63 +57,64 @@ class sfp_quad9(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ["INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE"]- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):- return ["MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME",- "MALICIOUS_COHOST"]+ return [+ "MALICIOUS_INTERNET_NAME",+ "MALICIOUS_AFFILIATE_INTERNET_NAME",+ "MALICIOUS_COHOST"]- def queryAddr(self, qaddr):+ def query(self, qry):res = dns.resolver.Resolver()res.nameservers = ["9.9.9.9"]try:- addrs = res.query(qaddr)- self.sf.debug("Addresses returned: " + str(addrs))+ addrs = res.query(qry)+ self.sf.debug(f"Addresses returned: {addrs}")except BaseException:- self.sf.debug(f"Unable to resolve {qaddr}")+ self.sf.debug(f"Unable to resolve {qry}")return Falseif addrs:return Truereturn False- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.dataparentEvent = event- resolved = Falseself.sf.debug(f"Received event, {eventName}, from {srcModuleName}")if eventData in self.results:return None+self.results[eventData] = True# Check that it resolves first, as it becomes a valid# malicious host only if NOT resolved by Quad9.- try:- if self.sf.resolveHost(eventData):- resolved = True- except BaseException:+ if not self.sf.resolveHost(eventData):+ return None++ found = self.query(eventData)++ # Host was found, not blocked+ if found:return None- if resolved:- found = self.queryAddr(eventData)- typ = "MALICIOUS_" + eventName- if eventName == "CO_HOSTED_SITE":- typ = "MALICIOUS_COHOST"- if not found:- evt = SpiderFootEvent(typ, "Blocked by Quad9 [" + eventData + "]\n" +\- "<SFURL>https://quad9.net/result/?url=" + eventData + "</SFURL>",- self.__name__, parentEvent)- self.notifyListeners(evt)+ typ = "MALICIOUS_" + eventName++ if eventName == "CO_HOSTED_SITE":+ typ = "MALICIOUS_COHOST"++ evt = SpiderFootEvent(+ typ,+ f"Blocked by Quad9 [{eventData}]\n<SFURL>https://quad9.net/result/?url={eventData}</SFURL>",+ self.__name__, parentEvent+ )+ self.notifyListeners(evt)# End of sfp_quad9 classdiff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.pyindex 7cdacbac..3bb0d838 100644--- a/modules/sfp_slideshare.py+++ b/modules/sfp_slideshare.py@@ -84,17 +84,18 @@ class sfp_slideshare(SpiderFootPlugin):network = eventData.split(": ")[0]url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:- self.sf.error("Unable to parse SOCIAL_MEDIA: "- + eventData + " (" + str(e) + ")", False)+ self.sf.error("Unable to parse SOCIAL_MEDIA: {eventData} ({e})", False)return Noneif not network == "SlideShare":- self.sf.debug("Skipping social network profile, " + url + \- ", as not a SlideShare profile")+ self.sf.debug(f"Skipping social network profile, {url}, as not a SlideShare profile")return None- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ url,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent']+ )if res['content'] is None:return None@@ -103,11 +104,10 @@ class sfp_slideshare(SpiderFootPlugin):human_name = self.extractMeta('slideshare:name', res['content'])if not human_name:- self.sf.debug(url + " is not a valid SlideShare profile")+ self.sf.debug(f"{url} is not a valid SlideShare profile")return None- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + \- human_name[0], self.__name__, event)+ e = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {human_name[0]}", self.__name__, event)self.notifyListeners(e)# Retrieve location (country)diff --git a/modules/sfp_snov.py b/modules/sfp_snov.pyindex fb52c72c..92adbae6 100644--- a/modules/sfp_snov.py+++ b/modules/sfp_snov.py@@ -191,7 +191,7 @@ class sfp_snov(SpiderFootPlugin):try:data = json.loads(data)- except:+ except BaseException:self.sf.debug("No email address found for target domain")breakdiff --git a/modules/sfp_social.py b/modules/sfp_social.pyindex cee4998f..56addc56 100644--- a/modules/sfp_social.py+++ b/modules/sfp_social.py@@ -22,9 +22,10 @@ regexps = dict({"Facebook": list(['.*facebook.com/([a-zA-Z0-9_]+$)']),"MySpace": list([r'https?://myspace.com/([a-zA-Z0-9_\.]+$)']),"YouTube": list(['.*youtube.com/([a-zA-Z0-9_]+$)']),- "Twitter": list(['.*twitter.com/([a-zA-Z0-9_]{1,15}$)',- '.*twitter.com/#!/([a-zA-Z0-9_]{1,15}$)'- ]),+ "Twitter": list([+ '.*twitter.com/([a-zA-Z0-9_]{1,15}$)',+ '.*twitter.com/#!/([a-zA-Z0-9_]{1,15}$)'+ ]),"SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)']),"Instagram": list([r'.*instagram.com/([a-zA-Z0-9_\.]+)/?$'])})@@ -40,14 +41,9 @@ class sfp_social(SpiderFootPlugin):'categories': ["Social Media"]}- # Default optionsopts = {}- # Option descriptionsoptdescs = {- # For each option in opts you should have a key/value pair here- # describing it. It will end up in the UI to explain the option- # to the end-user.}results = None@@ -60,18 +56,12 @@ class sfp_social(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for input- # * = be notified about all events.def watchedEvents(self):return ["LINKED_URL_EXTERNAL"]- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):return ["SOCIAL_MEDIA", "USERNAME"]- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -79,32 +69,32 @@ class sfp_social(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")- if eventData not in list(self.results.keys()):- self.results[eventData] = True- else:+ if eventData in list(self.results.keys()):return None+ self.results[eventData] = True+for regexpGrp in list(regexps.keys()):for regex in regexps[regexpGrp]:bits = re.match(regex, eventData, re.IGNORECASE)- if bits is not None:- self.sf.info("Matched " + regexpGrp + " in " + eventData)- evt = SpiderFootEvent("SOCIAL_MEDIA", regexpGrp + ": " + \- "<SFURL>" + eventData + "</SFURL>",- self.__name__, event)- self.notifyListeners(evt)- # Except for Google+, the link includes potential usernames- if regexpGrp != "Google+":- un = bits.group(1)- evt = SpiderFootEvent("USERNAME", str(un), self.__name__, event)- self.notifyListeners(evt)+ if not bits:+ continue++ self.sf.info(f"Matched {regexpGrp} in {eventData}")+ evt = SpiderFootEvent(+ "SOCIAL_MEDIA", f"{regexpGrp}: <SFURL>{eventData}</SFURL>",+ self.__name__,+ event+ )+ self.notifyListeners(evt)++ # Except for Google+, the link includes potential usernames+ if regexpGrp != "Google+":+ un = bits.group(1)+ evt = SpiderFootEvent("USERNAME", str(un), self.__name__, event)+ self.notifyListeners(evt)return None- # If you intend for this module to act on its own (e.g. not solely rely- # on events from other modules, then you need to have a start() method- # and within that method call self.checkForStop() to see if you've been- # politely asked by the controller to stop your activities (user abort.)-# End of sfp_social classdiff --git a/modules/sfp_spamhaus.py b/modules/sfp_spamhaus.pyindex da586415..069629e1 100644--- a/modules/sfp_spamhaus.py+++ b/modules/sfp_spamhaus.py@@ -87,14 +87,10 @@ class sfp_spamhaus(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ['IP_ADDRESS', 'AFFILIATE_IPADDR', 'NETBLOCK_OWNER','NETBLOCK_MEMBER']- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):return ["BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR","BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK"]@@ -151,7 +147,6 @@ class sfp_spamhaus(SpiderFootPlugin):return None- # Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -162,27 +157,26 @@ class sfp_spamhaus(SpiderFootPlugin):if eventData in self.results:return None+self.results[eventData] = Trueif eventName == 'NETBLOCK_OWNER':if not self.opts['netblocklookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:- self.sf.debug("Network size bigger than permitted: "- + str(IPNetwork(eventData).prefixlen) + " > "- + str(self.opts['maxnetblock']))- return None++ max_netblock = self.opts['maxnetblock']+ if IPNetwork(eventData).prefixlen < max_netblock:+ self.sf.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}")+ return Noneif eventName == 'NETBLOCK_MEMBER':if not self.opts['subnetlookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:- self.sf.debug("Network size bigger than permitted: "- + str(IPNetwork(eventData).prefixlen) + " > "- + str(self.opts['maxsubnet']))- return None++ max_subnet = self.opts['maxsubnet']+ if IPNetwork(eventData).prefixlen < max_subnet:+ self.sf.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}")+ return Noneif eventName.startswith("NETBLOCK_"):for addr in IPNetwork(eventData):diff --git a/modules/sfp_tool_whatweb.py b/modules/sfp_tool_whatweb.pyindex 7e6255d9..22e7e4b3 100644--- a/modules/sfp_tool_whatweb.py+++ b/modules/sfp_tool_whatweb.py@@ -113,7 +113,7 @@ class sfp_tool_whatweb(SpiderFootPlugin):aggression = 4if aggression < 1:aggression = 1- except:+ except BaseException:aggression = 1# Run WhatWeb@@ -131,7 +131,7 @@ class sfp_tool_whatweb(SpiderFootPlugin):p = Popen(args, stdout=PIPE, stderr=PIPE)stdout, stderr = p.communicate(input=None)except BaseException as e:- self.sf.error("Unable to run WhatWeb: " + str(e), False)+ self.sf.error(f"Unable to run WhatWeb: {e}", False)return Noneif p.returncode != 0:@@ -140,13 +140,13 @@ class sfp_tool_whatweb(SpiderFootPlugin):return Noneif not stdout:- self.sf.debug("WhatWeb returned no output for " + eventData)+ self.sf.debug(f"WhatWeb returned no output for {eventData}")return Nonetry:result_json = json.loads(stdout)except BaseException as e:- self.sf.error("Couldn't parse the JSON output of WhatWeb: " + str(e), False)+ self.sf.error(f"Couldn't parse the JSON output of WhatWeb: {e}", False)return Noneif len(result_json) == 0:diff --git a/modules/sfp_torch.py b/modules/sfp_torch.pyindex 9aad3fe2..c0cb37fd 100644--- a/modules/sfp_torch.py+++ b/modules/sfp_torch.py@@ -26,21 +26,18 @@ class sfp_torch(SpiderFootPlugin):'categories': ["Search Engines"]}- # Default optionsopts = {'fetchlinks': True,'pages': 20,'fullnames': True}- # Option descriptionsoptdescs = {'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.",'pages': "Number of results pages to iterate through.",'fullnames': "Search for human names?"}- # Targetresults = Nonedef setup(self, sfc, userOpts=dict()):@@ -50,13 +47,9 @@ class sfp_torch(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):return ["DOMAIN_NAME", "HUMAN_NAME", "EMAILADDR"]- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT", "SEARCH_ENGINE_WEB_CONTENT"]@@ -104,64 +97,81 @@ class sfp_torch(SpiderFootPlugin):pagecount += 1# Sites hosted on the domain- data = self.sf.fetchUrl("http://xmh57jrzrnw6insl.onion/" + formid + "/search.cgi?q=" + \- eventData.replace(" ", "%20") + "&cmd=Search!" + p,- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout'])+ data = self.sf.fetchUrl(+ f"http://xmh57jrzrnw6insl.onion/{formid}/search.cgi?q="+ + eventData.replace(" ", "%20") + "&cmd=Search!" + p,+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']+ )+if data is None or not data.get('content'):self.sf.info("No results returned from TORCH.")return Nonepagecontent = data['content']- if "No documents were found" not in data['content']:- # Submit the google results for analysis- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],- self.__name__, event)- self.notifyListeners(evt)+ if "No documents were found" in data['content']:+ return None- links = re.findall(r"\<DT\>\d+.\s+<a href=\"(.*?)\"\s+TARGET=\"_blank\"\>",- data['content'], re.IGNORECASE | re.DOTALL)+ # Submit the google results for analysis+ evt = SpiderFootEvent(+ "SEARCH_ENGINE_WEB_CONTENT",+ data['content'],+ self.__name__,+ event+ )+ self.notifyListeners(evt)++ links = re.findall(r"\<DT\>\d+.\s+<a href=\"(.*?)\"\s+TARGET=\"_blank\"\>",+ data['content'], re.IGNORECASE | re.DOTALL)++ for link in links:+ if link in self.results:+ continue++ self.results[link] = True+ self.sf.debug(f"Found a darknet mention: {link}")++ if self.sf.urlFQDN(link).endswith(".onion"):+ if self.checkForStop():+ return None++ if self.opts['fetchlinks']:+ res = self.sf.fetchUrl(+ link,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ verify=False+ )++ if res['content'] is None:+ self.sf.debug(f"Ignoring {link} as no data returned")+ continue++ if eventData not in res['content']:+ self.sf.debug(f"Ignoring {link} as no mention of {eventData}")+ continue++ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)+ self.notifyListeners(evt)++ try:+ startIndex = res['content'].index(eventData) - 120+ endIndex = startIndex + len(eventData) + 240+ except BaseException:+ self.sf.debug("String not found in content.")+ continue++ data = res['content'][startIndex:endIndex]+ evt = SpiderFootEvent(+ "DARKNET_MENTION_CONTENT", "..." + data + "...",+ self.__name__,+ evt+ )+ self.notifyListeners(evt)- for link in links:- if link in self.results:- continueelse:- self.results[link] = True- self.sf.debug("Found a darknet mention: " + link)- if self.sf.urlFQDN(link).endswith(".onion"):- if self.checkForStop():- return None- if self.opts['fetchlinks']:- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- verify=False)-- if res['content'] is None:- self.sf.debug("Ignoring " + link + " as no data returned")- continue-- if eventData not in res['content']:- self.sf.debug("Ignoring " + link + " as no mention of " + eventData)- continue- evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)- self.notifyListeners(evt)-- try:- startIndex = res['content'].index(eventData) - 120- endIndex = startIndex + len(eventData) + 240- except BaseException:- self.sf.debug("String not found in content.")- continue-- data = res['content'][startIndex:endIndex]- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",- self.__name__, evt)- self.notifyListeners(evt)-- else:- evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)- self.notifyListeners(evt)-+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)+ self.notifyListeners(evt)# End of sfp_torch classdiff --git a/modules/sfp_twilio.py b/modules/sfp_twilio.pyindex 1214004d..883a1469 100644--- a/modules/sfp_twilio.py+++ b/modules/sfp_twilio.py@@ -68,7 +68,6 @@ class sfp_twilio(SpiderFootPlugin):def watchedEvents(self):return ["PHONE_NUMBER"]- # What events this module producesdef producedEvents(self):return ["COMPANY_NAME", "RAW_RIR_DATA"]@@ -130,12 +129,11 @@ class sfp_twilio(SpiderFootPlugin):self.errorState = Truereturn None- # Don't look up stuff twiceif eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True++ self.results[eventData] = Truecontent = self.queryPhoneNumber(eventData)diff --git a/setup.cfg b/setup.cfgindex 36e257d9..8579e76f 100644--- a/setup.cfg+++ b/setup.cfg@@ -1,5 +1,5 @@[flake8]flake8-max-line-length = 120-ignore = F841 E501 W503 E131 E401 E302 W605 W504 E722 E502+ignore = F841 E501 W503 E131 E302 W605per-file-ignores =spiderfoot/__init__.py:F401