Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

flake8: resolve flake8 violations

Brendan Colesf58390038b9d
diff --git a/modules/sfp_apility.py b/modules/sfp_apility.py
index 16b14fa7..05ce9b07 100644
--- a/modules/sfp_apility.py
+++ b/modules/sfp_apility.py
@@ -111,7 +111,6 @@ class sfp_apility(SpiderFootPlugin):
return self.parseApiResponse(res)
-
# Query IP REST API
# https://apility.io/apidocs/#full-ip-address-reputation
# Note: currently unused
diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.py
index 44f077db..0e04df93 100644
--- a/modules/sfp_archiveorg.py
+++ b/modules/sfp_archiveorg.py
@@ -143,17 +143,17 @@ class sfp_archiveorg(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'])
- if res['content'] == None:
- self.sf.error("Unable to fetch " + url, False)
+ if res['content'] is None:
+ self.sf.error(f"Unable to fetch {url}", False)
continue
try:
ret = json.loads(res['content'])
except BaseException as e:
- self.sf.debug(f"Error processing JSON response: {e}")
+ self.sf.debug(f"Error processing JSON response from Archive.org: {e}")
ret = None
- if ret == None:
+ if not ret:
self.sf.error("Unable to process empty response from archive.org: {eventData}", False)
continue
diff --git a/modules/sfp_arin.py b/modules/sfp_arin.py
index 8d55cfd6..0ea585c7 100644
--- a/modules/sfp_arin.py
+++ b/modules/sfp_arin.py
@@ -121,7 +121,7 @@ class sfp_arin(SpiderFootPlugin):
j = json.loads(res['content'])
return j
except Exception as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
# Handle events sent to this module
diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.py
index be3947f3..6756c156 100644
--- a/modules/sfp_azureblobstorage.py
+++ b/modules/sfp_azureblobstorage.py
@@ -110,7 +110,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):
for site in sites:
if i >= self.opts['_maxthreads']:
data = self.threadSites(siteList)
- if data == None:
+ if data is None:
return res
for ret in list(data.keys()):
diff --git a/modules/sfp_bgpview.py b/modules/sfp_bgpview.py
index 849701a2..72a5ca7a 100644
--- a/modules/sfp_bgpview.py
+++ b/modules/sfp_bgpview.py
@@ -72,7 +72,7 @@ class sfp_bgpview(SpiderFootPlugin):
try:
json_data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response from BGPView: " + str(e))
+ self.sf.debug(f"Error processing JSON response from BGPView: {e}")
return None
if not json_data.get('status') == 'ok':
@@ -100,7 +100,7 @@ class sfp_bgpview(SpiderFootPlugin):
try:
json_data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response from BGPView: " + str(e))
+ self.sf.debug(f"Error processing JSON response from BGPView: {e}")
return None
if not json_data.get('status') == 'ok':
@@ -128,7 +128,7 @@ class sfp_bgpview(SpiderFootPlugin):
try:
json_data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response from BGPView: " + str(e))
+ self.sf.debug(f"Error processing JSON response from BGPView: {e}")
return None
if not json_data.get('status') == 'ok':
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.py
index 3ea310e2..5604df12 100644
--- a/modules/sfp_bingsearch.py
+++ b/modules/sfp_bingsearch.py
@@ -131,6 +131,4 @@ class sfp_bingsearch(SpiderFootPlugin):
)
self.notifyListeners(evt)
-
-
# End of sfp_bingsearch class
diff --git a/modules/sfp_botvrij.py b/modules/sfp_botvrij.py
index 32d2b004..0ff1d65a 100644
--- a/modules/sfp_botvrij.py
+++ b/modules/sfp_botvrij.py
@@ -16,7 +16,7 @@ import re
from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
- 'botvrij.eu Domain Blocklist': {
+ 'botvrij.eu Domain Blocklist': {
'id': '_botvrij',
'checks': ['domain'],
'url': 'https://www.botvrij.eu/data/blocklist/blocklist_full.csv',
diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.py
index b0f46f42..ea3fb8b5 100644
--- a/modules/sfp_citadel.py
+++ b/modules/sfp_citadel.py
@@ -108,7 +108,7 @@ class sfp_citadel(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.py
index 29f58829..a58183eb 100644
--- a/modules/sfp_cookie.py
+++ b/modules/sfp_cookie.py
@@ -67,7 +67,7 @@ class sfp_cookie(SpiderFootPlugin):
try:
jdata = json.loads(eventData)
- if jdata == None:
+ if jdata is None:
return None
except BaseException:
self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
diff --git a/modules/sfp_countryname.py b/modules/sfp_countryname.py
index 620a2dab..b6e9103c 100644
--- a/modules/sfp_countryname.py
+++ b/modules/sfp_countryname.py
@@ -47,6 +47,7 @@ class sfp_countryname(SpiderFootPlugin):
}
results = None
+
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
@@ -358,7 +359,6 @@ class sfp_countryname(SpiderFootPlugin):
self.sf.debug("Skipped invalid phone number: " + srcPhoneNumber)
return None
-
# Detect name of country from TLD of domain name
def detectCountryFromTLD(self, srcDomain):
@@ -495,9 +495,10 @@ class sfp_countryname(SpiderFootPlugin):
countryNames = set(countryNames)
for countryName in countryNames:
- if countryName == '' or countryName == None:
+ if not countryName:
continue
- self.sf.debug("Found country name: " + countryName)
+
+ self.sf.debug(f"Found country name: {countryName}")
evt = SpiderFootEvent(evttype, countryName, self.__name__, event)
if event.moduleDataSource:
diff --git a/modules/sfp_creditcard.py b/modules/sfp_creditcard.py
index f431703d..af9c65eb 100644
--- a/modules/sfp_creditcard.py
+++ b/modules/sfp_creditcard.py
@@ -26,7 +26,6 @@ class sfp_creditcard(SpiderFootPlugin):
# Default options.
opts = {
- # Options specific to this module
}
# Option descriptions.
diff --git a/modules/sfp_crt.py b/modules/sfp_crt.py
index 4c80ba11..6f64a44e 100644
--- a/modules/sfp_crt.py
+++ b/modules/sfp_crt.py
@@ -100,7 +100,7 @@ class sfp_crt(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if data is None or len(data) == 0:
diff --git a/modules/sfp_darksearch.py b/modules/sfp_darksearch.py
index 34d05eea..ee77c787 100644
--- a/modules/sfp_darksearch.py
+++ b/modules/sfp_darksearch.py
@@ -93,7 +93,7 @@ class sfp_darksearch(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.py
index 90d25a03..9431f320 100644
--- a/modules/sfp_digitaloceanspace.py
+++ b/modules/sfp_digitaloceanspace.py
@@ -70,9 +70,8 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
- if res['code'] not in ["301", "302", "200"] and \
- (res['content'] is None or "NoSuchBucket" in res['content']):
- self.sf.debug("Not a valid bucket: " + url)
+ if res['code'] not in ["301", "302", "200"] and (res['content'] is None or "NoSuchBucket" in res['content']):
+ self.sf.debug(f"Not a valid bucket: {url}")
else:
if "ListBucketResult" in res['content']:
with self.lock:
@@ -120,7 +119,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
for site in sites:
if i >= self.opts['_maxthreads']:
data = self.threadSites(siteList)
- if data == None:
+ if data is None:
return res
for ret in list(data.keys()):
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
index ce5e93e2..deafe9ae 100644
--- a/modules/sfp_dnsresolve.py
+++ b/modules/sfp_dnsresolve.py
@@ -92,17 +92,17 @@ class sfp_dnsresolve(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
return [
- # Events that need some kind of DNS treatment
- "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER",
- "IP_ADDRESS", "IPV6_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",
- # Events that may contain hostnames in their content
- "TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS",
- "CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAN_WHOIS", "NETBLOCK_WHOIS",
- "LEAKSITE_CONTENT", "RAW_DNS_RECORDS", "RAW_FILE_META_DATA",
- "RAW_RIR_DATA", "SEARCH_ENGINE_WEB_CONTENT", "SIMILARDOMAIN_WHOIS",
- "SSL_CERTIFICATE_RAW", "SSL_CERTIFICATE_ISSUED", "TCP_PORT_OPEN_BANNER",
- "WEBSERVER_BANNER", "WEBSERVER_HTTPHEADERS"
- ]
+ # Events that need some kind of DNS treatment
+ "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER",
+ "IP_ADDRESS", "IPV6_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",
+ # Events that may contain hostnames in their content
+ "TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS",
+ "CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAN_WHOIS", "NETBLOCK_WHOIS",
+ "LEAKSITE_CONTENT", "RAW_DNS_RECORDS", "RAW_FILE_META_DATA",
+ "RAW_RIR_DATA", "SEARCH_ENGINE_WEB_CONTENT", "SIMILARDOMAIN_WHOIS",
+ "SSL_CERTIFICATE_RAW", "SSL_CERTIFICATE_ISSUED", "TCP_PORT_OPEN_BANNER",
+ "WEBSERVER_BANNER", "WEBSERVER_HTTPHEADERS"
+ ]
# What events this module produces
# This is to support the end user in selecting modules based on events
diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.py
index 23ebc6fa..fa12d071 100644
--- a/modules/sfp_duckduckgo.py
+++ b/modules/sfp_duckduckgo.py
@@ -39,12 +39,12 @@ class sfp_duckduckgo(SpiderFootPlugin):
# Default options
opts = {
- "affiliatedomains": True
+ "affiliatedomains": True
}
# Option descriptions
optdescs = {
- "affiliatedomains": "For affiliates, look up the domain name, not the hostname. This will usually return more meaningful information about the affiliate."
+ "affiliatedomains": "For affiliates, look up the domain name, not the hostname. This will usually return more meaningful information about the affiliate."
}
results = None
@@ -69,7 +69,6 @@ class sfp_duckduckgo(SpiderFootPlugin):
"AFFILIATE_DESCRIPTION_CATEGORY",
"AFFILIATE_DESCRIPTION_ABSTRACT"]
-
def handleEvent(self, event):
eventName = event.eventType
eventData = event.data
@@ -80,17 +79,17 @@ class sfp_duckduckgo(SpiderFootPlugin):
return None
if eventData in self.results:
- self.sf.debug("Already did a search for " + eventData + ", skipping.")
+ self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
url = "https://api.duckduckgo.com/?q=" + eventData + "&format=json&pretty=1"
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot")
- if res['content'] == None:
- self.sf.error("Unable to fetch " + url, False)
+ if res['content'] is None:
+ self.sf.error(f"Unable to fetch {url}", False)
return None
try:
@@ -99,38 +98,44 @@ class sfp_duckduckgo(SpiderFootPlugin):
self.sf.error(f"Error processing JSON response from DuckDuckGo: {e}", False)
return None
- if ret['Heading'] == "":
- self.sf.debug("No DuckDuckGo information for " + eventData)
+ if not ret['Heading']:
+ self.sf.debug(f"No DuckDuckGo information for {eventData}")
return None
- # Submit the bing results for analysis
+ # Submit the DuckDuckGo results for analysis
evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", res['content'],
self.__name__, event)
self.notifyListeners(evt)
- if 'AbstractText' in ret:
- name = "DESCRIPTION_ABSTRACT"
+ abstract_text = ret.get('AbstractText')
+ if abstract_text:
+ event_type = "DESCRIPTION_ABSTRACT"
+
if "AFFILIATE" in eventName:
- name = "AFFILIATE_" + name
+ event_type = "AFFILIATE_" + event_type
- evt = SpiderFootEvent(name, ret['AbstractText'],
- self.__name__, event)
+ evt = SpiderFootEvent(event_type, str(abstract_text), self.__name__, event)
self.notifyListeners(evt)
- if 'RelatedTopics' in ret:
- name = "DESCRIPTION_CATEGORY"
+ related_topics = ret.get('RelatedTopics')
+ if related_topics:
+ event_type = "DESCRIPTION_CATEGORY"
+
if "AFFILIATE" in eventName:
- name = "AFFILIATE_" + name
+ event_type = "AFFILIATE_" + event_type
+
+ for topic in related_topics:
+ if not isinstance(topic, dict):
+ self.sf.debug("No category text found from DuckDuckGo.")
+ continue
+
+ category = topic.get('Text')
- for item in ret['RelatedTopics']:
- cat = None
- if 'Text' in item:
- cat = item['Text']
- if cat == None or cat == "":
+ if not category:
self.sf.debug("No category text found from DuckDuckGo.")
continue
- evt = SpiderFootEvent(name, cat, self.__name__, event)
+ evt = SpiderFootEvent(event_type, category, self.__name__, event)
self.notifyListeners(evt)
# End of sfp_duckduckgo class
diff --git a/modules/sfp_emailrep.py b/modules/sfp_emailrep.py
index 1896102d..3aed268e 100644
--- a/modules/sfp_emailrep.py
+++ b/modules/sfp_emailrep.py
@@ -114,7 +114,7 @@ class sfp_emailrep(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.py
index 26303d5c..022b463f 100644
--- a/modules/sfp_fortinet.py
+++ b/modules/sfp_fortinet.py
@@ -81,14 +81,12 @@ class sfp_fortinet(SpiderFootPlugin):
def watchedEvents(self):
return ["IP_ADDRESS", "AFFILIATE_IPADDR"]
-
# What events this module produces
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR"]
-
# Check the regexps to see whether the content indicates maliciousness
def contentMalicious(self, content, goodregex, badregex):
# First, check for the bad indicators
diff --git a/modules/sfp_fraudguard.py b/modules/sfp_fraudguard.py
index b53d420f..751ba9d2 100644
--- a/modules/sfp_fraudguard.py
+++ b/modules/sfp_fraudguard.py
@@ -122,10 +122,8 @@ class sfp_fraudguard(SpiderFootPlugin):
self.sf.error(f"Error processing JSON response from Fraudguard.io: {e}", False)
return None
- #print str(info)
return info
-
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
@@ -146,18 +144,17 @@ class sfp_fraudguard(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxnetblock']))
- return None
+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
+ self.sf.debug("Network size bigger than permitted: "
+ + str(IPNetwork(eventData).prefixlen) + " > "
+ + str(self.opts['maxnetblock']))
+ return None
qrylist = list()
rtype = ""
diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.py
index 2f00e60b..ffcfe20e 100644
--- a/modules/sfp_fringeproject.py
+++ b/modules/sfp_fringeproject.py
@@ -81,7 +81,7 @@ class sfp_fringeproject(SpiderFootPlugin):
try:
json_data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response from Fringe Project: " + str(e))
+ self.sf.debug(f"Error processing JSON response from Fringe Project: {e}")
return None
data = json_data.get('results')
diff --git a/modules/sfp_fsecure_riddler.py b/modules/sfp_fsecure_riddler.py
index 96e87d46..5f5ce6c1 100644
--- a/modules/sfp_fsecure_riddler.py
+++ b/modules/sfp_fsecure_riddler.py
@@ -101,7 +101,7 @@ class sfp_fsecure_riddler(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response from F-Secure Riddler: " + str(e))
+ self.sf.debug(f"Error processing JSON response from F-Secure Riddler: {e}")
return None
try:
@@ -130,7 +130,6 @@ class sfp_fsecure_riddler(SpiderFootPlugin):
'Content-Type': 'application/json',
}
-
res = self.sf.fetchUrl('https://riddler.io/api/search',
postData=json.dumps(params),
headers=headers,
@@ -150,7 +149,7 @@ class sfp_fsecure_riddler(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response from F-Secure Riddler: " + str(e))
+ self.sf.debug(f"Error processing JSON response from F-Secure Riddler: {e}")
return None
if not data:
diff --git a/modules/sfp_fullcontact.py b/modules/sfp_fullcontact.py
index de2faafe..6e4f9175 100644
--- a/modules/sfp_fullcontact.py
+++ b/modules/sfp_fullcontact.py
@@ -125,7 +125,7 @@ class sfp_fullcontact(SpiderFootPlugin):
self.sf.debug("Fullcontact.co record found but too old.")
return None
except Exception as e:
- self.sf.error("Error processing JSON response from fullcontact.com: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from fullcontact.com: {e}", False)
return None
return ret
diff --git a/modules/sfp_github.py b/modules/sfp_github.py
index cd870a93..e581b004 100644
--- a/modules/sfp_github.py
+++ b/modules/sfp_github.py
@@ -68,13 +68,15 @@ class sfp_github(SpiderFootPlugin):
repo_info = None
# Get repos matching the name
- if item.get('name') == None:
+ if item.get('name') is None:
self.sf.debug("Incomplete Github information found (name).")
return None
- if item.get('html_url') == None:
+
+ if item.get('html_url') is None:
self.sf.debug("Incomplete Github information found (url).")
return None
- if item.get('description') == None:
+
+ if item.get('description') is None:
self.sf.debug("Incomplete Github information found (description).")
return None
@@ -88,10 +90,10 @@ class sfp_github(SpiderFootPlugin):
eventData = event.data
if eventData in self.results:
- self.sf.debug("Already did a search for " + eventData + ", skipping.")
+ self.sf.debug(f"Already did a search for {eventData}, skipping.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
# Extract name and location from profile
if eventName == "SOCIAL_MEDIA":
@@ -99,22 +101,25 @@ class sfp_github(SpiderFootPlugin):
network = eventData.split(": ")[0]
url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")
except BaseException as e:
- self.sf.error("Unable to parse SOCIAL_MEDIA: " +
- eventData + " (" + str(e) + ")", False)
+ self.sf.error(f"Unable to parse SOCIAL_MEDIA: {eventData} ({e})", False)
return None
if not network == "Github":
- self.sf.debug("Skipping social network profile, " + url + ", as not a Github profile")
+ self.sf.debug(f"Skipping social network profile, {url}, as not a GitHub profile")
return None
try:
- bits = url.split("/")
- name = bits[len(bits)-1]
+ urlParts = url.split("/")
+ username = urlParts[len(urlParts)-1]
except BaseException:
- self.sf.debug("Couldn't get a username out of " + url)
+ self.sf.debug(f"Couldn't get a username out of {url}")
return None
- res = self.sf.fetchUrl("https://api.github.com/users/" + name, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
+ res = self.sf.fetchUrl(
+ f"https://api.github.com/users/{username}",
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent']
+ )
if res['content'] is None:
return None
@@ -126,14 +131,16 @@ class sfp_github(SpiderFootPlugin):
return None
if not json_data.get('login'):
- self.sf.debug(name + " is not a valid Github profile")
+ self.sf.debug(f"{username} is not a valid GitHub profile")
return None
- if not json_data.get('name'):
- self.sf.debug(name + " is not a valid Github profile")
+ full_name = json_data.get('name')
+
+ if not full_name:
+ self.sf.debug(f"{username} is not a valid GitHub profile")
return None
- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + json_data['name'], self.__name__, event)
+ e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: {full_name}", self.__name__, event)
self.notifyListeners(e)
location = json_data.get('location')
@@ -142,7 +149,7 @@ class sfp_github(SpiderFootPlugin):
return None
if len(location) < 3 or len(location) > 100:
- self.sf.debug("Skipping likely invalid location: " + location)
+ self.sf.debug(f"Skipping likely invalid location: {location}")
return None
e = SpiderFootEvent("GEOINFO", location, self.__name__, event)
@@ -151,24 +158,27 @@ class sfp_github(SpiderFootPlugin):
return None
if eventName == "DOMAIN_NAME":
- name = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
- if not name:
+ username = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
+ if not username:
return None
if eventName == "USERNAME":
- name = eventData
+ username = eventData
- self.sf.debug("Looking at " + name)
+ self.sf.debug(f"Looking at {username}")
failed = False
# Get all the repositories based on direct matches with the
# name identified
- url = "https://api.github.com/search/repositories?q=" + name
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
-
- if res['content'] == None:
- self.sf.error("Unable to fetch " + url, False)
+ url = f"https://api.github.com/search/repositories?q={username}"
+ res = self.sf.fetchUrl(
+ url,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent']
+ )
+
+ if res['content'] is None:
+ self.sf.error(f"Unable to fetch {url}", False)
failed = True
try:
@@ -178,54 +188,56 @@ class sfp_github(SpiderFootPlugin):
ret = None
if ret is None:
- self.sf.error(f"Unable to process empty response from Github for: {name}", False)
+ self.sf.error(f"Unable to process empty response from Github for: {username}", False)
failed = True
if not failed:
if ret.get('total_count', "0") == "0" or len(ret['items']) == 0:
- self.sf.debug("No Github information for " + name)
+ self.sf.debug(f"No Github information for {username}")
failed = True
if not failed:
for item in ret['items']:
repo_info = self.buildRepoInfo(item)
- if repo_info != None:
- if self.opts['namesonly'] and name != item['name']:
+ if repo_info is not None:
+ if self.opts['namesonly'] and username != item['name']:
continue
- evt = SpiderFootEvent("PUBLIC_CODE_REPO", repo_info,
- self.__name__, event)
+ evt = SpiderFootEvent("PUBLIC_CODE_REPO", repo_info, self.__name__, event)
self.notifyListeners(evt)
# Now look for users matching the name found
failed = False
- url = "https://api.github.com/search/users?q=" + name
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
-
- if res['content'] == None:
- self.sf.error("Unable to fetch " + url, False)
+ url = f"https://api.github.com/search/users?q={username}"
+ res = self.sf.fetchUrl(
+ url,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent']
+ )
+
+ if res['content'] is None:
+ self.sf.error(f"Unable to fetch {url}", False)
failed = True
if not failed:
try:
ret = json.loads(res['content'])
- if ret == None:
- self.sf.error(f"Unable to process empty response from Github for: {name}", False)
+ if ret is None:
+ self.sf.error(f"Unable to process empty response from Github for: {username}", False)
failed = True
except BaseException:
- self.sf.error(f"Unable to process invalid response from Github for: {name}", False)
+ self.sf.error(f"Unable to process invalid response from Github for: {username}", False)
failed = True
if not failed:
if ret.get('total_count', "0") == "0" or len(ret['items']) == 0:
- self.sf.debug("No Github information for " + name)
+ self.sf.debug("No Github information for " + username)
failed = True
if not failed:
- # For each user matching the name, get their repos
+ # For each user matching the username, get their repos
for item in ret['items']:
- if item.get('repos_url') == None:
+ if item.get('repos_url') is None:
self.sf.debug("Incomplete Github information found (repos_url).")
continue
@@ -233,8 +245,8 @@ class sfp_github(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'])
- if res['content'] == None:
- self.sf.error("Unable to fetch " + url, False)
+ if res['content'] is None:
+ self.sf.error(f"Unable to fetch {url}", False)
continue
try:
@@ -243,9 +255,8 @@ class sfp_github(SpiderFootPlugin):
self.sf.error(f"Invalid JSON returned from Github: {e}", False)
continue
- if repret == None:
- self.sf.error("Unable to process empty response from Github for: " + \
- name, False)
+ if repret is None:
+ self.sf.error(f"Unable to process empty response from Github for: {username}", False)
continue
for item in repret:
@@ -254,10 +265,10 @@ class sfp_github(SpiderFootPlugin):
continue
repo_info = self.buildRepoInfo(item)
- if repo_info != None:
- if self.opts['namesonly'] and item['name'] != name:
+ if repo_info is not None:
+ if self.opts['namesonly'] and item['name'] != username:
continue
- if eventName == "USERNAME" and "/" + name + "/" not in item.get('html_url', ''):
+ if eventName == "USERNAME" and "/" + username + "/" not in item.get('html_url', ''):
continue
evt = SpiderFootEvent("PUBLIC_CODE_REPO", repo_info,
diff --git a/modules/sfp_googleobjectstorage.py b/modules/sfp_googleobjectstorage.py
index 0210c800..b9fac248 100644
--- a/modules/sfp_googleobjectstorage.py
+++ b/modules/sfp_googleobjectstorage.py
@@ -68,9 +68,8 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
- if res['code'] not in ["301", "302", "200"] and \
- (res['content'] is None or "NoSuchBucket" in res['content']):
- self.sf.debug("Not a valid bucket: " + url)
+ if res['code'] not in ["301", "302", "200"] and (res['content'] is None or "NoSuchBucket" in res['content']):
+ self.sf.debug(f"Not a valid bucket: {url}")
else:
if "ListBucketResult" in res['content']:
with self.lock:
@@ -118,7 +117,7 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
for site in sites:
if i >= self.opts['_maxthreads']:
data = self.threadSites(siteList)
- if data == None:
+ if data is None:
return res
for ret in list(data.keys()):
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index 0c7b930e..578e362d 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -93,7 +93,7 @@ class sfp_gravatar(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if data.get('entry') is None or len(data.get('entry')) == 0:
@@ -183,7 +183,6 @@ class sfp_gravatar(SpiderFootPlugin):
self.notifyListeners(evt)
self.reportedUsers[v] = True
-
if data.get('accounts') is not None:
for account in data.get('accounts'):
url = account.get('url')
diff --git a/modules/sfp_grep_app.py b/modules/sfp_grep_app.py
index 6757150e..6abd0d59 100644
--- a/modules/sfp_grep_app.py
+++ b/modules/sfp_grep_app.py
@@ -87,7 +87,7 @@ class sfp_grep_app(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.py
index 3d2995d2..68fc5850 100644
--- a/modules/sfp_hackertarget.py
+++ b/modules/sfp_hackertarget.py
@@ -262,7 +262,6 @@ class sfp_hackertarget(SpiderFootPlugin):
else:
hosts.append(strdata)
-
for host in set(hosts):
if self.getTarget().matches(host, includeChildren=True, includeParents=True):
evt_type = 'INTERNET_NAME'
diff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.py
index e8bce9fa..026c756d 100644
--- a/modules/sfp_haveibeenpwned.py
+++ b/modules/sfp_haveibeenpwned.py
@@ -134,7 +134,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
self.results[eventData] = True
data = self.query(eventData)
- if data == None:
+ if data is None:
return None
for n in data:
diff --git a/modules/sfp_honeypot.py b/modules/sfp_honeypot.py
index a8e91e62..4d371347 100644
--- a/modules/sfp_honeypot.py
+++ b/modules/sfp_honeypot.py
@@ -137,10 +137,9 @@ class sfp_honeypot(SpiderFootPlugin):
eventName = parentEvent.eventType
try:
- lookup = self.opts['api_key'] + "." + \
- self.reverseAddr(qaddr) + ".dnsbl.httpbl.org"
+ lookup = self.opts['api_key'] + f".{self.reverseAddr(qaddr)}.dnsbl.httpbl.org"
- self.sf.debug("Checking Honeypot: " + lookup)
+ self.sf.debug(f"Checking Honeypot: {lookup}")
addrs = self.sf.resolveHost(lookup)
if not addrs:
return None
diff --git a/modules/sfp_hostio.py b/modules/sfp_hostio.py
index 85a9497c..e02d168b 100644
--- a/modules/sfp_hostio.py
+++ b/modules/sfp_hostio.py
@@ -178,7 +178,6 @@ class sfp_hostio(SpiderFootPlugin):
self.notifyListeners(geo_info_evt)
found = True
-
related = data.get("related")
if related and isinstance(related, dict):
email_section = related.get("email")
diff --git a/modules/sfp_hunter.py b/modules/sfp_hunter.py
index 7aa61705..ade4280d 100644
--- a/modules/sfp_hunter.py
+++ b/modules/sfp_hunter.py
@@ -160,7 +160,7 @@ class sfp_hunter(SpiderFootPlugin):
self.notifyListeners(e)
if 'first_name' in email and 'last_name' in email:
- if email['first_name'] != None and email['last_name'] != None:
+ if email['first_name'] is not None and email['last_name'] is not None:
n = email['first_name'] + " " + email['last_name']
e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + n,
self.__name__, event)
@@ -170,7 +170,7 @@ class sfp_hunter(SpiderFootPlugin):
return None
data = self.query(eventData, rescount, 10)
- if data == None:
+ if data is None:
return None
if "data" not in data:
return None
diff --git a/modules/sfp_iban.py b/modules/sfp_iban.py
index d33e1598..0657708f 100644
--- a/modules/sfp_iban.py
+++ b/modules/sfp_iban.py
@@ -26,7 +26,6 @@ class sfp_iban(SpiderFootPlugin):
# Default options.
opts = {
- # Options specific to this module
}
# Option descriptions.
diff --git a/modules/sfp_iknowwhatyoudownload.py b/modules/sfp_iknowwhatyoudownload.py
index b0e358f4..a4c729b3 100644
--- a/modules/sfp_iknowwhatyoudownload.py
+++ b/modules/sfp_iknowwhatyoudownload.py
@@ -87,8 +87,8 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):
ret = None
retdata = None
- base = "https://api.antitor.com/history/peer/?ip="
- url = base + qry + "&days=" + str(self.opts['daysback'])
+ url = "https://api.antitor.com/history/peer/?ip="
+ url += qry + "&days=" + str(self.opts['daysback'])
url += "&key=" + self.opts['api_key']
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")
@@ -100,7 +100,7 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from iknowwhatyoudownload.com: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from iknowwhatyoudownload.com: {e}", False)
return None
if 'error' in ret:
@@ -109,17 +109,16 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):
self.sf.error("The number of days you have configured is not accepted. If you have the demo key, try 30 days or less.", False)
return None
- if 'contents' in ret:
- if len(ret['contents']) > 0:
- retdata = "<SFURL>https://iknowwhatyoudownload.com/en/peer/?ip=" + qry + "</SFURL>\n"
- for d in ret['contents']:
- retdata += d['torrent']['name'] + \
- " (" + d.get("endDate", "Date unknown") + ")\n"
- else:
- return None
- else:
+ if 'contents' not in ret:
+ return None
+
+ if not len(ret['contents']):
return None
+ retdata = "<SFURL>https://iknowwhatyoudownload.com/en/peer/?ip=" + qry + "</SFURL>\n"
+ for d in ret['contents']:
+ retdata += d['torrent']['name'] + " (" + d.get("endDate", "Date unknown") + ")\n"
+
return retdata
# Handle events sent to this module
@@ -142,14 +141,15 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
data = self.query(eventData)
- if data == None:
+
+ if not data:
return None
- else:
- e = SpiderFootEvent("MALICIOUS_IPADDR", data, self.__name__, event)
- self.notifyListeners(e)
+
+ e = SpiderFootEvent("MALICIOUS_IPADDR", data, self.__name__, event)
+ self.notifyListeners(e)
# End of sfp_iknowwhatyoudownload class
diff --git a/modules/sfp_instagram.py b/modules/sfp_instagram.py
index b52a4c7c..6ca548bb 100644
--- a/modules/sfp_instagram.py
+++ b/modules/sfp_instagram.py
@@ -72,7 +72,7 @@ class sfp_instagram(SpiderFootPlugin):
try:
data = json.loads(json_data[0])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_intelx.py b/modules/sfp_intelx.py
index 30e5055d..d3a449f5 100644
--- a/modules/sfp_intelx.py
+++ b/modules/sfp_intelx.py
@@ -138,7 +138,7 @@ class sfp_intelx(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from IntelligenceX: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from IntelligenceX: {e}", False)
self.errorState = True
return None
@@ -278,7 +278,4 @@ class sfp_intelx(SpiderFootPlugin):
e = SpiderFootEvent("DOMAIN_NAME", val, self.__name__, event)
self.notifyListeners(e)
-
-
-
# End of sfp_intelx class
diff --git a/modules/sfp_ipstack.py b/modules/sfp_ipstack.py
index 4dcd5ba8..9c5ab50b 100644
--- a/modules/sfp_ipstack.py
+++ b/modules/sfp_ipstack.py
@@ -103,7 +103,7 @@ class sfp_ipstack(SpiderFootPlugin):
try:
hostip = json.loads(res['content'])
- if 'success' in hostip and hostip['success'] == False:
+ if 'success' in hostip and hostip['success'] is False:
self.sf.error("Invalid ipstack.com API key or usage limits exceeded.", False)
self.errorState = True
return None
diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.py
index d70614c9..59b93656 100644
--- a/modules/sfp_multiproxy.py
+++ b/modules/sfp_multiproxy.py
@@ -86,7 +86,6 @@ class sfp_multiproxy(SpiderFootPlugin):
def watchedEvents(self):
return ["IP_ADDRESS", "AFFILIATE_IPADDR"]
-
# What events this module produces
# This is to support the end user in selecting modules based on events
# produced.
diff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.py
index 2a825632..cab48d03 100644
--- a/modules/sfp_neutrinoapi.py
+++ b/modules/sfp_neutrinoapi.py
@@ -198,7 +198,6 @@ class sfp_neutrinoapi(SpiderFootPlugin):
evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
self.notifyListeners(evt)
-
if eventName == 'IP_ADDRESS':
data = self.queryIpInfo(eventData)
diff --git a/modules/sfp_numverify.py b/modules/sfp_numverify.py
index 481e824c..01ce97e9 100644
--- a/modules/sfp_numverify.py
+++ b/modules/sfp_numverify.py
@@ -118,7 +118,7 @@ class sfp_numverify(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if data.get('error') is not None:
diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.py
index cc6a988d..970bb510 100644
--- a/modules/sfp_opencorporates.py
+++ b/modules/sfp_opencorporates.py
@@ -93,7 +93,7 @@ class sfp_opencorporates(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if 'results' not in data:
@@ -123,7 +123,7 @@ class sfp_opencorporates(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if 'results' not in data:
@@ -131,7 +131,6 @@ class sfp_opencorporates(SpiderFootPlugin):
return data['results']
-
# Extract company address, previous names, and officer names
def extractCompanyDetails(self, company, sevt):
diff --git a/modules/sfp_openstreetmap.py b/modules/sfp_openstreetmap.py
index 25d7af3d..62bbd8f6 100644
--- a/modules/sfp_openstreetmap.py
+++ b/modules/sfp_openstreetmap.py
@@ -82,7 +82,7 @@ class sfp_openstreetmap(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response: " + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_psbdmp.py b/modules/sfp_psbdmp.py
index 5f951125..4b18ae40 100644
--- a/modules/sfp_psbdmp.py
+++ b/modules/sfp_psbdmp.py
@@ -90,19 +90,19 @@ class sfp_psbdmp(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from psbdmp.cc: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from psbdmp.cc: {e}", False)
return None
ids = list()
- if 'count' in ret:
- if ret['count'] > 0:
- for d in ret['data']:
- ids.append("https://pastebin.com/" + d['id'])
- else:
- return None
- else:
+ if 'count' not in ret:
+ return None
+
+ if ret['count'] <= 0:
return None
+ for d in ret['data']:
+ ids.append("https://pastebin.com/" + d['id'])
+
return ids
# Handle events sent to this module
@@ -121,7 +121,7 @@ class sfp_psbdmp(SpiderFootPlugin):
self.results[eventData] = True
data = self.query(eventData)
- if data == None:
+ if data is None:
return None
for n in data:
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
index 5fb4e8ec..e2edd009 100644
--- a/modules/sfp_s3bucket.py
+++ b/modules/sfp_s3bucket.py
@@ -70,8 +70,7 @@ class sfp_s3bucket(SpiderFootPlugin):
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
- if res['code'] not in ["301", "302", "200"] and \
- (res['content'] is None or "NoSuchBucket" in res['content']):
+ if res['code'] not in ["301", "302", "200"] and (res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
if "ListBucketResult" in res['content']:
@@ -120,13 +119,13 @@ class sfp_s3bucket(SpiderFootPlugin):
for site in sites:
if i >= self.opts['_maxthreads']:
data = self.threadSites(siteList)
- if data == None:
+ if data is None:
return res
for ret in list(data.keys()):
if data[ret]:
# bucket:filecount
- res.append(ret + ":" + str(data[ret]))
+ res.append(f"{ret}:{data[ret]}")
i = 0
siteList = list()
diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.py
index d212ac73..2a6d4341 100644
--- a/modules/sfp_scylla.py
+++ b/modules/sfp_scylla.py
@@ -107,7 +107,7 @@ class sfp_scylla(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_snov.py b/modules/sfp_snov.py
index 428122d3..af26107b 100644
--- a/modules/sfp_snov.py
+++ b/modules/sfp_snov.py
@@ -68,7 +68,6 @@ class sfp_snov(SpiderFootPlugin):
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
-
# What events is this module interested in for input
# For a list of all events, check sfdb.py.
def watchedEvents(self):
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 30b03881..a5e419df 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -218,14 +218,14 @@ class sfp_spider(SpiderFootPlugin):
sendcontent = False
if sendcontent:
- if httpresult['content'] != None:
+ if httpresult['content'] is not None:
event = SpiderFootEvent("TARGET_WEB_CONTENT", httpresult['content'],
self.__name__, parentEvent)
event.actualSource = url
self.notifyListeners(event)
hdr = httpresult['headers']
- if hdr != None:
+ if hdr is not None:
event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(hdr, ensure_ascii=False),
self.__name__, parentEvent)
event.actualSource = url
diff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.py
index 66ae8d52..8f26e01e 100644
--- a/modules/sfp_strangeheaders.py
+++ b/modules/sfp_strangeheaders.py
@@ -78,7 +78,7 @@ class sfp_strangeheaders(SpiderFootPlugin):
try:
jdata = json.loads(eventData)
- if jdata == None:
+ if jdata is None:
return None
except BaseException:
self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
diff --git a/modules/sfp_template.py b/modules/sfp_template.py
index d672c694..ff6a88c8 100644
--- a/modules/sfp_template.py
+++ b/modules/sfp_template.py
@@ -319,7 +319,6 @@ class sfp_template(SpiderFootPlugin):
+ str(self.opts['maxnetblock']))
return None
-
# When handling netblocks/subnets, assuming the user set
# netblocklookup/subnetlookup to True, we need to expand it
# to the IPs for looking up.
diff --git a/modules/sfp_venmo.py b/modules/sfp_venmo.py
index 8cae6cc1..4126deb6 100644
--- a/modules/sfp_venmo.py
+++ b/modules/sfp_venmo.py
@@ -72,7 +72,7 @@ class sfp_venmo(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
json_data = data.get('data')
diff --git a/modules/sfp_viewdns.py b/modules/sfp_viewdns.py
index bbca63e3..f1ca549f 100644
--- a/modules/sfp_viewdns.py
+++ b/modules/sfp_viewdns.py
@@ -136,7 +136,7 @@ class sfp_viewdns(SpiderFootPlugin):
# We are at the last or only page
self.accum.extend(r.get(responsekey, []))
except Exception as e:
- self.sf.error("Error processing JSON response from ViewDNS.info: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from ViewDNS.info: {e}", False)
return None
# Handle events sent to this module
diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.py
index a9570629..01210f24 100644
--- a/modules/sfp_vxvault.py
+++ b/modules/sfp_vxvault.py
@@ -16,12 +16,12 @@ import re
from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
- 'VXVault Malicious URL List': {
+ 'VXVault Malicious URL List': {
'id': '_vxvault',
'checks': ['ip', 'domain'],
'url': 'http://vxvault.net/URL_List.php',
'regex': r'.*\/{0}/.*'
- }
+ }
}
diff --git a/modules/sfp_webanalytics.py b/modules/sfp_webanalytics.py
index 45163cda..62dd89b4 100644
--- a/modules/sfp_webanalytics.py
+++ b/modules/sfp_webanalytics.py
@@ -235,7 +235,6 @@ class sfp_webanalytics(SpiderFootPlugin):
evt.moduleDataSource = datasource
self.notifyListeners(evt)
-
# TeamViewer SSO Verification
# https://community.teamviewer.com/t5/Knowledge-Base/Single-Sign-On-SSO/ta-p/30784
matches = re.findall(r'teamviewer-sso-verification=([a-f0-9]{32})$', eventData.strip(), re.IGNORECASE)
diff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.py
index 246af5da..149e5b68 100644
--- a/modules/sfp_webserver.py
+++ b/modules/sfp_webserver.py
@@ -60,8 +60,8 @@ class sfp_webserver(SpiderFootPlugin):
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventSource in self.results:
return None
- else:
- self.results[eventSource] = True
+
+ self.results[eventSource] = True
if not self.getTarget().matches(self.sf.urlFQDN(eventSource)):
self.sf.debug("Not collecting web server information for external sites.")
@@ -69,7 +69,7 @@ class sfp_webserver(SpiderFootPlugin):
try:
jdata = json.loads(eventData)
- if jdata == None:
+ if jdata is None:
return None
except BaseException:
self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
diff --git a/modules/sfp_whatcms.py b/modules/sfp_whatcms.py
index 2efe6a0b..671f1218 100644
--- a/modules/sfp_whatcms.py
+++ b/modules/sfp_whatcms.py
@@ -129,7 +129,7 @@ class sfp_whatcms(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug('Error processing JSON response: ' + str(e))
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
result = data.get('result')
diff --git a/modules/sfp_whoisology.py b/modules/sfp_whoisology.py
index e35dfb97..1fdf2e48 100644
--- a/modules/sfp_whoisology.py
+++ b/modules/sfp_whoisology.py
@@ -80,8 +80,6 @@ class sfp_whoisology(SpiderFootPlugin):
# Search Whoisology
def query(self, qry, querytype):
- info = None
-
url = "https://whoisology.com/api?auth=" + self.opts['api_key'] + "&request=flat"
url += "&field=" + querytype + "&value=" + qry + "&level=Registrant|Admin|Tec|Billing|Other"
@@ -94,22 +92,22 @@ class sfp_whoisology(SpiderFootPlugin):
return None
if res['content'] is None:
- self.sf.info("No Whoisology info found for " + qry)
+ self.sf.info(f"No Whoisology info found for {qry}")
return None
try:
info = json.loads(res['content'])
- if info.get("domains") == None:
+ if info.get("domains") is None:
self.sf.error("Error querying Whoisology: " + info.get("status_reason", "Unknown"), False)
return None
if len(info.get("domains", [])) == 0:
- self.sf.debug("No data found in Whoisology for " + qry)
+ self.sf.debug(f"No data found in Whoisology for {qry}")
return None
- else:
- return info.get('domains')
+
+ return info.get('domains')
except Exception as e:
- self.sf.error("Error processing JSON response from Whoisology: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from Whoisology: {e}", False)
return None
# Handle events sent to this module
diff --git a/modules/sfp_wigle.py b/modules/sfp_wigle.py
index acf028fc..78995e3c 100644
--- a/modules/sfp_wigle.py
+++ b/modules/sfp_wigle.py
@@ -86,17 +86,24 @@ class sfp_wigle(SpiderFootPlugin):
return ["WIFI_ACCESS_POINT"]
def getcoords(self, qry):
- url = "https://api.wigle.net/api/v2/network/geocode?" + \
- urllib.parse.urlencode({'addresscode': qry.encode('utf-8', errors='replace')})
+ params = {
+ 'addresscode': qry.encode('utf-8', errors='replace')
+ }
hdrs = {
- "Accept": "application/json",
- "Authorization": "Basic " + self.opts['api_key_encoded']
- }
+ "Accept": "application/json",
+ "Authorization": "Basic " + self.opts['api_key_encoded']
+ }
+
+ res = self.sf.fetchUrl(
+ "https://api.wigle.net/api/v2/network/geocode?" + urllib.parse.urlencode(params),
+ timeout=30,
+ useragent="SpiderFoot",
+ headers=hdrs
+ )
- res = self.sf.fetchUrl(url, timeout=30,
- useragent="SpiderFoot", headers=hdrs)
if res['code'] == "404" or not res['content']:
return None
+
if "too many queries" in res['content']:
self.sf.error("Wigle.net query limit reached for the day.", False)
return None
@@ -111,25 +118,37 @@ class sfp_wigle(SpiderFootPlugin):
return None
def getnetworks(self, coords):
- url = "https://api.wigle.net/api/v2/network/search?onlymine=false&" + \
- "latrange1=" + str(coords[0]) + "&latrange2=" + str(coords[1]) + \
- "&longrange1=" + str(coords[2]) + "&longrange2=" + str(coords[3]) + \
- "&freenet=false&paynet=false&variance=" + self.opts['variance']
+ params = {
+ 'onlymine': 'false',
+ 'latrange1': str(coords[0]),
+ 'latrange2': str(coords[1]),
+ 'longrange1': str(coords[2]),
+ 'longrange2': str(coords[3]),
+ 'freenet': 'false',
+ 'paynet': 'false',
+ 'variance': self.opts['variance']
+ }
if self.opts['days_limit'] != "0":
dt = datetime.datetime.now() - datetime.timedelta(days=int(self.opts['days_limit']))
date_calc = dt.strftime("%Y%m%d")
- url += "&lastupdt=" + date_calc
+ params['lastupdt'] = date_calc
hdrs = {
- "Accept": "application/json",
- "Authorization": "Basic " + self.opts['api_key_encoded']
- }
+ "Accept": "application/json",
+ "Authorization": "Basic " + self.opts['api_key_encoded']
+ }
+
+ res = self.sf.fetchUrl(
+ "https://api.wigle.net/api/v2/network/search?" + urllib.parse.urlencode(params),
+ timeout=30,
+ useragent="SpiderFoot",
+ headers=hdrs
+ )
- res = self.sf.fetchUrl(url, timeout=30,
- useragent="SpiderFoot", headers=hdrs)
if res['code'] == "404" or not res['content']:
return None
+
if "too many queries" in res['content']:
self.sf.error("Wigle.net query limit reached for the day.", False)
return None
@@ -137,17 +156,19 @@ class sfp_wigle(SpiderFootPlugin):
ret = list()
try:
info = json.loads(res['content'])
+
if len(info.get('results', [])) == 0:
return None
+
for r in info['results']:
if None not in [r['ssid'], r['netid']]:
ret.append(r['ssid'] + " (Net ID: " + r['netid'] + ")")
+
return ret
except Exception as e:
self.sf.error(f"Error processing JSON response from WiGLE: {e}", False)
return None
-
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
@@ -168,8 +189,8 @@ class sfp_wigle(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
coords = self.getcoords(eventData)
if not coords:
diff --git a/modules/sfp_wikileaks.py b/modules/sfp_wikileaks.py
index 83b00fb4..ac4f0a99 100644
--- a/modules/sfp_wikileaks.py
+++ b/modules/sfp_wikileaks.py
@@ -82,15 +82,15 @@ class sfp_wikileaks(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
if self.opts['external']:
external = "True"
else:
external = ""
- if self.opts['daysback'] != None and self.opts['daysback'] != 0:
+ if self.opts['daysback'] is not None and self.opts['daysback'] != 0:
newDate = datetime.datetime.now() - datetime.timedelta(days=int(self.opts['daysback']))
maxDate = newDate.strftime("%Y-%m-%d")
else:
@@ -100,6 +100,7 @@ class sfp_wikileaks(SpiderFootPlugin):
wlurl = "https://search.wikileaks.org/?query=%22" + qdata + "%22" + \
"&released_date_start=" + maxDate + "&include_external_sources=" + \
external + "&new_search=True&order_by=most_relevant#results"
+
res = self.sf.fetchUrl(wlurl)
if res['content'] is None:
self.sf.error("Unable to fetch Wikileaks content.", False)
diff --git a/modules/sfp_wikipediaedits.py b/modules/sfp_wikipediaedits.py
index d4ad9d73..6087590d 100644
--- a/modules/sfp_wikipediaedits.py
+++ b/modules/sfp_wikipediaedits.py
@@ -86,10 +86,10 @@ class sfp_wikipediaedits(SpiderFootPlugin):
params["month"] = dt.strftime("%m")
res = self.sf.fetchUrl(
- "https://en.wikipedia.org/w/api.php?%s" % urllib.parse.urlencode(params),
- timeout=self.opts['_fetchtimeout'],
- useragent="SpiderFoot"
- )
+ "https://en.wikipedia.org/w/api.php?%s" % urllib.parse.urlencode(params),
+ timeout=self.opts['_fetchtimeout'],
+ useragent="SpiderFoot"
+ )
if res['code'] in ["404", "403", "500"]:
return None
diff --git a/setup.cfg b/setup.cfg
index 27867363..817132d8 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,3 +1,3 @@
[flake8]
flake8-max-line-length = 120
-ignore = F841 E501 W503 E131 E401 E201 E302 E265 W605 W504 E121 E226 E722 E127 E741 E126 E711 E303 E502 E123 E301 E125 E262 E712
+ignore = F841 E501 W503 E131 E401 E201 E302 E265 W605 W504 E121 E226 E722 E127 E741 E502