Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

resolve flake8 violations

Brendan Colesa0d8e9eee084
diff --git a/modules/sfp__stor_db.py b/modules/sfp__stor_db.py
index fd39cae3..65d59a1f 100644
--- a/modules/sfp__stor_db.py
+++ b/modules/sfp__stor_db.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin
+from sflib import SpiderFootPlugin
class sfp__stor_db(SpiderFootPlugin):
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py
index 423e872c..531240ad 100644
--- a/modules/sfp__stor_stdout.py
+++ b/modules/sfp__stor_stdout.py
@@ -10,7 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin
+from sflib import SpiderFootPlugin
import json
@@ -27,7 +27,7 @@ class sfp__stor_stdout(SpiderFootPlugin):
# Default options
opts = {
- "_format": "tab", # tab, csv, json
+ "_format": "tab", # tab, csv, json
"_requested": [],
"_showonlyrequested": False,
"_stripnewline": False,
diff --git a/modules/sfp_abusech.py b/modules/sfp_abusech.py
index dbf07e88..65f30ce6 100644
--- a/modules/sfp_abusech.py
+++ b/modules/sfp_abusech.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'abuse.ch Zeus Tracker (Domain)': {
@@ -58,8 +58,8 @@ class sfp_abusech(SpiderFootPlugin):
'name': "abuse.ch",
'summary': "Check if a host/domain, IP or netblock is malicious according to abuse.ch.",
'flags': [],
- 'useCases': [ "Passive", "Investigate" ],
- 'categories': [ "Reputation Systems" ],
+ 'useCases': ["Passive", "Investigate"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.abuse.ch",
'model': "FREE_AUTH_UNLIMITED",
@@ -184,8 +184,7 @@ class sfp_abusech(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_abuseipdb.py b/modules/sfp_abuseipdb.py
index 7bc091d0..c17d0d39 100644
--- a/modules/sfp_abuseipdb.py
+++ b/modules/sfp_abuseipdb.py
@@ -12,7 +12,7 @@
import re
from netaddr import IPAddress, IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'AbuseIPDB': {
@@ -28,9 +28,9 @@ class sfp_abuseipdb(SpiderFootPlugin):
meta = {
'name': "AbuseIPDB",
'summary': "Check if an IP address is malicious according to AbuseIPDB.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Passive", "Investigate" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Passive", "Investigate"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.abuseipdb.com",
'model': "FREE_AUTH_LIMITED",
@@ -145,8 +145,7 @@ class sfp_abuseipdb(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 22042eaa..54d51ceb 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -16,7 +16,7 @@ import threading
from queue import Queue, Empty as QueueEmpty
import json
import random
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_accounts(SpiderFootPlugin):
"""Account Finder:Footprint,Passive:Social Media:slow:Look for possible associated accounts on nearly 200 websites like Ebay, Slashdot, reddit, etc."""
@@ -24,8 +24,8 @@ class sfp_accounts(SpiderFootPlugin):
meta = {
'name': "Account Finder",
'summary': "Look for possible associated accounts on nearly 200 websites like Ebay, Slashdot, reddit, etc.",
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Social Media" ]
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Social Media"]
}
# Default options
@@ -85,7 +85,7 @@ class sfp_accounts(SpiderFootPlugin):
try:
self.sites = [site for site in json.loads(content)['sites'] if site['valid']]
except BaseException as e:
- self.sf.error("Unable to parse social media accounts list.", False)
+ self.sf.error(f"Unable to parse social media accounts list: {e}", False)
self.errorState = True
return None
@@ -150,8 +150,8 @@ class sfp_accounts(SpiderFootPlugin):
site = queue.get(timeout=0.1)
try:
self.checkSite(username, site)
- except Exception as ex:
- self.sf.debug(f'thread {threading.current_thread().name} exception {ex}')
+ except Exception as e:
+ self.sf.debug(f'thread {threading.current_thread().name} exception {e}')
except QueueEmpty:
return
@@ -240,7 +240,7 @@ class sfp_accounts(SpiderFootPlugin):
self.distrustedChecked = True
if eventName == "HUMAN_NAME":
- names = [ eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".") ]
+ names = [eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".")]
for name in names:
users.append(name)
@@ -259,7 +259,6 @@ class sfp_accounts(SpiderFootPlugin):
users.append(eventData)
for user in users:
- adduser = True
if user in self.opts['_genericusers'].split(","):
self.sf.debug(user + " is a generic account name, skipping.")
continue
diff --git a/modules/sfp_adblock.py b/modules/sfp_adblock.py
index 69c51030..3ddd36a9 100644
--- a/modules/sfp_adblock.py
+++ b/modules/sfp_adblock.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import adblockparser
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_adblock(SpiderFootPlugin):
@@ -21,8 +21,8 @@ class sfp_adblock(SpiderFootPlugin):
meta = {
'name': "AdBlock Check",
'summary': "Check if linked pages would be blocked by AdBlock Plus.",
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://adblockplus.org/",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_ahmia.py b/modules/sfp_ahmia.py
index 11e3ab69..d88fffd7 100644
--- a/modules/sfp_ahmia.py
+++ b/modules/sfp_ahmia.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import re
@@ -21,8 +21,8 @@ class sfp_ahmia(SpiderFootPlugin):
meta = {
'name': "Ahmia",
'summary': "Search Tor 'Ahmia' search engine for mentions of the target domain.",
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Search Engines" ],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://ahmia.fi/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -80,7 +80,6 @@ class sfp_ahmia(SpiderFootPlugin):
def handleEvent(self, event):
eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':
@@ -105,8 +104,7 @@ class sfp_ahmia(SpiderFootPlugin):
if self.checkForStop():
return None
- links = re.findall("redirect_url=(.[^\"]+)\"",
- data['content'], re.IGNORECASE | re.DOTALL)
+ links = re.findall("redirect_url=(.[^\"]+)\"", data['content'], re.IGNORECASE | re.DOTALL)
reported = False
for link in links:
@@ -137,7 +135,7 @@ class sfp_ahmia(SpiderFootPlugin):
try:
startIndex = res['content'].index(eventData) - 120
endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
+ except BaseException:
self.sf.debug("String not found in content.")
continue
diff --git a/modules/sfp_alienvault.py b/modules/sfp_alienvault.py
index cd7ad3ce..02965cf6 100644
--- a/modules/sfp_alienvault.py
+++ b/modules/sfp_alienvault.py
@@ -14,7 +14,7 @@ import json
from datetime import datetime
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_alienvault(SpiderFootPlugin):
"""AlienVault OTX:Investigate,Passive:Reputation Systems:apikey:Obtain information from AlienVault Open Threat Exchange (OTX)"""
@@ -22,9 +22,9 @@ class sfp_alienvault(SpiderFootPlugin):
meta = {
'name': "AlienVault OTX",
'summary': "Obtain information from AlienVault Open Threat Exchange (OTX)",
- 'flags': [ "apikey"],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://otx.alienvault.com/",
'model': "FREE_AUTH_LIMITED",
@@ -105,10 +105,9 @@ class sfp_alienvault(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK" ]
+ return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK"]
def query(self, qry, querytype):
- ret = None
targettype = "hostname"
if ":" in qry:
@@ -141,12 +140,11 @@ class sfp_alienvault(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from AlienVault OTX.", False)
+ self.sf.error(f"Error processing JSON response from AlienVault OTX: {e}", False)
return None
return info
-
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
@@ -221,7 +219,7 @@ class sfp_alienvault(SpiderFootPlugin):
if self.opts['age_limit_days'] > 0 and last_ts < age_limit_ts:
self.sf.debug("Record found but too old, skipping.")
continue
- except BaseException as e:
+ except BaseException:
self.sf.debug("Couldn't parse date from AlienVault so assuming it's OK.")
e = SpiderFootEvent(evtType, host, self.__name__, event)
self.notifyListeners(e)
@@ -258,7 +256,7 @@ class sfp_alienvault(SpiderFootPlugin):
if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:
self.sf.debug("Record found but too old, skipping.")
continue
- except BaseException as e:
+ except BaseException:
self.sf.debug("Couldn't parse date from AlienVault so assuming it's OK.")
# For netblocks, we need to create the IP address event so that
diff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.py
index fd4a5a84..3b6f269e 100644
--- a/modules/sfp_alienvaultiprep.py
+++ b/modules/sfp_alienvaultiprep.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'AlienVault IP Reputation Database': {
@@ -31,8 +31,8 @@ class sfp_alienvaultiprep(SpiderFootPlugin):
meta = {
'name': "AlienVault IP Reputation",
'summary': "Check if an IP or netblock is malicious according to the AlienVault IP Reputation database.",
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://cybersecurity.att.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -126,8 +126,7 @@ class sfp_alienvaultiprep(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_api_recon_dev.py b/modules/sfp_api_recon_dev.py
index 4c1e3286..a5bb7519 100644
--- a/modules/sfp_api_recon_dev.py
+++ b/modules/sfp_api_recon_dev.py
@@ -13,7 +13,7 @@
import json
import time
import urllib
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_api_recon_dev(SpiderFootPlugin):
"""api.recon.dev:Footprint,Investigate,Passive:Passive DNS::Search api.recon.dev for subdomains."""
@@ -21,9 +21,9 @@ class sfp_api_recon_dev(SpiderFootPlugin):
meta = {
'name': "api.recon.dev",
'summary': "Search api.recon.dev for subdomains.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"]
}
opts = {
@@ -55,7 +55,7 @@ class sfp_api_recon_dev(SpiderFootPlugin):
def queryDomain(self, qry):
headers = {
- "Accept" : "application/json"
+ "Accept": "application/json"
}
params = {
'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
@@ -90,7 +90,7 @@ class sfp_api_recon_dev(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
# returns list of results; 'null' when no results; or dict when there's an error
diff --git a/modules/sfp_apility.py b/modules/sfp_apility.py
index 6b054f2a..0ca3880e 100644
--- a/modules/sfp_apility.py
+++ b/modules/sfp_apility.py
@@ -13,7 +13,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_apility(SpiderFootPlugin):
"""Apility:Footprint,Investigate,Passive:Reputation Systems:apikey:Search Apility API for IP address and domain reputation."""
@@ -21,9 +21,9 @@ class sfp_apility(SpiderFootPlugin):
meta = {
'name': "Apility",
'summary': "Search Apility API for IP address and domain reputation.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ]
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Reputation Systems"]
}
# Default options
@@ -151,7 +151,7 @@ class sfp_apility(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.py
index af3f8263..0c28c08f 100644
--- a/modules/sfp_archiveorg.py
+++ b/modules/sfp_archiveorg.py
@@ -13,7 +13,7 @@
import datetime
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_archiveorg(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_archiveorg(SpiderFootPlugin):
meta = {
'name': "Archive.org",
'summary': "Identifies historic versions of interesting files/pages from the Wayback Machine.",
- 'flags': [ "slow" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["slow"],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://archive.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -150,11 +150,11 @@ class sfp_archiveorg(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except BaseException as e:
+ self.sf.debug(f"Error processing JSON response: {e}")
ret = None
if ret == None:
- self.sf.error("Unable to process empty response from archive.org: " + \
- eventData, False)
+ self.sf.error("Unable to process empty response from archive.org: {eventData}", False)
continue
if len(ret['archived_snapshots']) < 1:
diff --git a/modules/sfp_arin.py b/modules/sfp_arin.py
index d5d9256a..86fc2b5f 100644
--- a/modules/sfp_arin.py
+++ b/modules/sfp_arin.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_arin(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_arin(SpiderFootPlugin):
meta = {
'name': "ARIN",
'summary': "Queries ARIN registry for contact information.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Public Registries" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Public Registries"],
'dataSource': {
'website': "https://www.arin.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -80,7 +80,7 @@ class sfp_arin(SpiderFootPlugin):
if url in self.memCache:
res = self.memCache[url]
else:
- head = { "Accept": "application/json" }
+ head = {"Accept": "application/json"}
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'], headers=head)
if res['content'] is not None:
@@ -92,7 +92,6 @@ class sfp_arin(SpiderFootPlugin):
# Owner information about an AS
def query(self, qtype, value):
- ownerinfo = dict()
url = "https://whois.arin.net/rest/"
if qtype == "domain":
diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.py
index 10842b76..fdfa0fa5 100644
--- a/modules/sfp_azureblobstorage.py
+++ b/modules/sfp_azureblobstorage.py
@@ -13,7 +13,7 @@
import threading
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_azureblobstorage(SpiderFootPlugin):
"""Azure Blob Finder:Footprint,Passive:Crawling and Scanning::Search for potential Azure blobs associated with the target and attempt to list their contents."""
@@ -21,9 +21,9 @@ class sfp_azureblobstorage(SpiderFootPlugin):
meta = {
'name': "Azure Blob Finder",
'summary': "Search for potential Azure blobs associated with the target and attempt to list their contents.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Crawling and Scanning" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Crawling and Scanning"],
'dataSource': {
'website': "https://azure.microsoft.com/en-in/services/storage/blobs/",
'model': "FREE_NOAUTH_UNLIMITED"
@@ -72,7 +72,6 @@ class sfp_azureblobstorage(SpiderFootPlugin):
self.s3results[url] = True
def threadSites(self, siteList):
- ret = list()
self.s3results = dict()
running = True
i = 0
@@ -145,7 +144,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):
self.notifyListeners(evt)
return None
- targets = [ eventData.replace('.', '') ]
+ targets = [eventData.replace('.', '')]
kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
if kw:
targets.append(kw)
diff --git a/modules/sfp_badipscom.py b/modules/sfp_badipscom.py
index a71dc908..1e1d6b63 100644
--- a/modules/sfp_badipscom.py
+++ b/modules/sfp_badipscom.py
@@ -10,7 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_badipscom(SpiderFootPlugin):
"""badips.com:Investigate,Passive:Reputation Systems::Check if an IP address is malicious according to BadIPs.com."""
@@ -18,9 +18,9 @@ class sfp_badipscom(SpiderFootPlugin):
meta = {
'name': "badips.com",
'summary': "Check if an IP address is malicious according to BadIPs.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.badips.com/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_badpackets.py b/modules/sfp_badpackets.py
index e40b66bf..b646e604 100644
--- a/modules/sfp_badpackets.py
+++ b/modules/sfp_badpackets.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
from netaddr import IPNetwork
import urllib.request, urllib.parse, urllib.error
import json
@@ -22,9 +22,9 @@ class sfp_badpackets(SpiderFootPlugin):
meta = {
'name': "Bad Packets",
'summary': "Obtain information about any malicious activities involving IP addresses found",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://badpackets.net",
'model': "COMMERCIAL_ONLY",
@@ -140,7 +140,7 @@ class sfp_badpackets(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from Bad Packets.", False)
+ self.sf.error(f"Error processing JSON response from Bad Packets: {e}", False)
return None
return data
@@ -167,28 +167,24 @@ class sfp_badpackets(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxnetblock']))
- return None
+
+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
+ self.sf.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {self.opts['maxnetblock']}")
+ return None
if eventName == 'NETBLOCK_MEMBER':
if not self.opts['subnetlookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxsubnet']))
- return None
+
+ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:
+ self.sf.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {self.opts['maxsubnet']}")
+ return None
qrylist = list()
if eventName.startswith("NETBLOCK_"):
@@ -246,7 +242,7 @@ class sfp_badpackets(SpiderFootPlugin):
continue
if maliciousIP:
- maliciousIPDesc = "Bad Packets [ " + str(maliciousIP) + " ]\n"
+ maliciousIPDesc = "Bad Packets [" + str(maliciousIP) + "]\n"
try:
category = record.get('tags')[0].get('category')
diff --git a/modules/sfp_bambenek.py b/modules/sfp_bambenek.py
index e989025a..58197716 100644
--- a/modules/sfp_bambenek.py
+++ b/modules/sfp_bambenek.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'Bambenek C&C IP List': {
@@ -36,9 +36,9 @@ class sfp_bambenek(SpiderFootPlugin):
meta = {
'name': "Bambenek C&C List",
'summary': "Check if a host/domain or IP appears on Bambenek Consulting's C&C tracker lists.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.bambenekconsulting.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -140,8 +140,7 @@ class sfp_bambenek(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_bgpview.py b/modules/sfp_bgpview.py
index 08691919..be8b5d1c 100644
--- a/modules/sfp_bgpview.py
+++ b/modules/sfp_bgpview.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_bgpview(SpiderFootPlugin):
"""BGPView:Investigate,Footprint,Passive:Search Engines::Obtain network information from BGPView API."""
@@ -20,9 +20,9 @@ class sfp_bgpview(SpiderFootPlugin):
meta = {
'name': "BGPView",
'summary': "Obtain network information from BGPView API.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Footprint", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Footprint", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://bgpview.io/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.py
index e3338543..fdd35deb 100644
--- a/modules/sfp_binaryedge.py
+++ b/modules/sfp_binaryedge.py
@@ -13,7 +13,7 @@
import json
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_binaryedge(SpiderFootPlugin):
"""BinaryEdge:Footprint,Investigate,Passive:Search Engines:apikey:Obtain information from BinaryEdge.io's Internet scanning systems about breaches, vulerabilities, torrents and passive DNS."""
@@ -21,9 +21,9 @@ class sfp_binaryedge(SpiderFootPlugin):
meta = {
'name': "BinaryEdge",
'summary': "Obtain information from BinaryEdge.io's Internet scanning systems about breaches, vulerabilities, torrents and passive DNS.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.binaryedge.io/",
'model': "FREE_AUTH_LIMITED",
@@ -104,7 +104,7 @@ class sfp_binaryedge(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
return ["IP_ADDRESS", "DOMAIN_NAME", "EMAILADDR",
- "NETBLOCK_OWNER", "NETBLOCK_MEMBER" ]
+ "NETBLOCK_OWNER", "NETBLOCK_MEMBER"]
# What events this module produces
def producedEvents(self):
@@ -114,7 +114,6 @@ class sfp_binaryedge(SpiderFootPlugin):
"CO_HOSTED_SITE", "MALICIOUS_IPADDR"]
def query(self, qry, querytype, page=1):
- ret = None
retarr = list()
if self.errorState:
@@ -141,7 +140,7 @@ class sfp_binaryedge(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot", headers=headers)
- if res['code'] in [ "429", "500" ]:
+ if res['code'] in ["429", "500"]:
self.sf.error("BinaryEdge.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)
self.errorState = True
return None
@@ -157,7 +156,7 @@ class sfp_binaryedge(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from BinaryEdge.io.", False)
+ self.sf.error(f"Error processing JSON response from BinaryEdge.io: {e}", False)
return None
if info.get('page') and info['total'] > info.get('pagesize', 100) * info.get('page', 0):
@@ -407,7 +406,7 @@ class sfp_binaryedge(SpiderFootPlugin):
banner = prec['result']['data']['service']['banner']
e = SpiderFootEvent(evtbtype, banner, self.__name__, ev)
self.notifyListeners(e)
- except BaseException as e:
+ except BaseException:
self.sf.debug("No banner information found.")
for addr in qrylist:
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.py
index ec086b7b..3ea310e2 100644
--- a/modules/sfp_bingsearch.py
+++ b/modules/sfp_bingsearch.py
@@ -9,7 +9,7 @@
# Copyright: (c) Steve Micallef 2013
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_bingsearch(SpiderFootPlugin):
@@ -18,9 +18,9 @@ class sfp_bingsearch(SpiderFootPlugin):
meta = {
'name': "Bing",
'summary': "Obtain information from bing to identify sub-domains and links.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.bing.com/",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.py
index 7608b15e..077cacee 100644
--- a/modules/sfp_bingsharedip.py
+++ b/modules/sfp_bingsharedip.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_bingsharedip(SpiderFootPlugin):
@@ -20,9 +20,9 @@ class sfp_bingsharedip(SpiderFootPlugin):
meta = {
'name': "Bing (Shared IPs)",
'summary': "Search Bing for hosts sharing the same IP.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.bing.com/",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_binstring.py b/modules/sfp_binstring.py
index 2d5587a4..10d50a78 100644
--- a/modules/sfp_binstring.py
+++ b/modules/sfp_binstring.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import string
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_binstring(SpiderFootPlugin):
"""Binary String Extractor:Footprint:Content Analysis:errorprone:Attempt to identify strings in binary content."""
@@ -19,9 +19,9 @@ class sfp_binstring(SpiderFootPlugin):
meta = {
'name': "Binary String Extractor",
'summary': "Attempt to identify strings in binary content.",
- 'flags': [ "errorprone" ],
- 'useCases': [ "Footprint" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': ["errorprone"],
+ 'useCases': ["Footprint"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -31,8 +31,8 @@ class sfp_binstring(SpiderFootPlugin):
'maxfilesize': 1000000,
'usedict': True,
'fileexts': ['png', 'gif', 'jpg', 'jpeg', 'tiff', 'tif',
- 'ico', 'flv', 'mp4', 'mp3', 'avi', 'mpg',
- 'mpeg', 'dat', 'mov', 'swf', 'exe', 'bin'],
+ 'ico', 'flv', 'mp4', 'mp3', 'avi', 'mpg',
+ 'mpeg', 'dat', 'mov', 'swf', 'exe', 'bin'],
'filterchars': '#}{|%^&*()=+,;[]~'
}
diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.py
index 4a870efa..cbd06cec 100644
--- a/modules/sfp_bitcoin.py
+++ b/modules/sfp_bitcoin.py
@@ -14,7 +14,7 @@
import re
import codecs
from hashlib import sha256
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_bitcoin(SpiderFootPlugin):
"""Bitcoin Finder:Footprint,Investigate,Passive:Content Analysis::Identify bitcoin addresses in scraped webpages."""
@@ -22,9 +22,9 @@ class sfp_bitcoin(SpiderFootPlugin):
meta = {
'name': "Bitcoin Finder",
'summary': "Identify bitcoin addresses in scraped webpages.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_blockchain.py b/modules/sfp_blockchain.py
index 90f528ae..ccafa82a 100644
--- a/modules/sfp_blockchain.py
+++ b/modules/sfp_blockchain.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_blockchain(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_blockchain(SpiderFootPlugin):
meta = {
'name': "Blockchain",
'summary': "Queries blockchain.info to find the balance of identified bitcoin wallet addresses.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Public Registries" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Public Registries"],
'dataSource': {
'website': "https://www.blockchain.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -90,7 +90,7 @@ class sfp_blockchain(SpiderFootPlugin):
data = json.loads(res['content'])
balance = float(data[eventData]['final_balance']) / 100000000
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
evt = SpiderFootEvent("BITCOIN_BALANCE", str(balance) + " BTC", self.__name__, event)
diff --git a/modules/sfp_blocklistde.py b/modules/sfp_blocklistde.py
index f4a147e5..b7ae104a 100644
--- a/modules/sfp_blocklistde.py
+++ b/modules/sfp_blocklistde.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'blocklist.de List': {
@@ -30,9 +30,9 @@ class sfp_blocklistde(SpiderFootPlugin):
meta = {
'name': "blocklist.de",
'summary': "Check if a netblock or IP is malicious according to blocklist.de.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.blocklist.de/en/index.html",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -126,8 +126,7 @@ class sfp_blocklistde(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_botscout.py b/modules/sfp_botscout.py
index c2008fc2..e0165837 100644
--- a/modules/sfp_botscout.py
+++ b/modules/sfp_botscout.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_botscout(SpiderFootPlugin):
@@ -20,9 +20,9 @@ class sfp_botscout(SpiderFootPlugin):
meta = {
'name': "BotScout",
'summary': "Searches botscout.com's database of spam-bot IPs and e-mail addresses.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Passive", "Investigate" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Passive", "Investigate"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://botscout.com/",
'model': "FREE_NOAUTH_LIMITED",
diff --git a/modules/sfp_botvrij.py b/modules/sfp_botvrij.py
index d9f43bb6..32d2b004 100644
--- a/modules/sfp_botvrij.py
+++ b/modules/sfp_botvrij.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'botvrij.eu Domain Blocklist': {
@@ -31,9 +31,9 @@ class sfp_botvrij(SpiderFootPlugin):
meta = {
'name': "botvrij.eu",
'summary': "Check if a domain is malicious according to botvrij.eu.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ]
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"]
}
# Default options
@@ -110,8 +110,7 @@ class sfp_botvrij(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_builtwith.py b/modules/sfp_builtwith.py
index 3d87fc6c..32b22fa2 100644
--- a/modules/sfp_builtwith.py
+++ b/modules/sfp_builtwith.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_builtwith(SpiderFootPlugin):
"""BuiltWith:Footprint,Investigate,Passive:Search Engines:apikey:Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more."""
@@ -20,9 +20,9 @@ class sfp_builtwith(SpiderFootPlugin):
meta = {
'name': "BuiltWith",
'summary': "Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://builtwith.com/",
'model': "FREE_AUTH_LIMITED",
@@ -80,21 +80,18 @@ class sfp_builtwith(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "DOMAIN_NAME" ]
+ return ["DOMAIN_NAME"]
# What events this module produces
def producedEvents(self):
- return [ "INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
- "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME",
- "CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID" ]
+ return ["INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
+ "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME",
+ "CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID"]
def queryRelationships(self, t):
- ret = None
+ url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}"
- url = "https://api.builtwith.com/rv1/api.json?LOOKUP=" + t + "&KEY=" + self.opts['api_key']
-
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent="SpiderFoot")
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")
if res['code'] == "404":
return None
@@ -103,20 +100,15 @@ class sfp_builtwith(SpiderFootPlugin):
return None
try:
- ret = json.loads(res['content'])['Relationships']
+ return json.loads(res['content'])['Relationships']
except Exception as e:
- self.sf.error("Error processing JSON response from builtwith.com: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from builtwith.com: {e}", False)
return None
- return ret
-
def queryDomainInfo(self, t):
- ret = None
+ url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}"
- url = "https://api.builtwith.com/v11/api.json?LOOKUP=" + t + "&KEY=" + self.opts['api_key']
-
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent="SpiderFoot")
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")
if res['code'] == "404":
return None
@@ -125,13 +117,11 @@ class sfp_builtwith(SpiderFootPlugin):
return None
try:
- ret = json.loads(res['content'])['Results'][0]
+ return json.loads(res['content'])['Results'][0]
except Exception as e:
- self.sf.error("Error processing JSON response from builtwith.com: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from builtwith.com: {e}", False)
return None
- return ret
-
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
diff --git a/modules/sfp_callername.py b/modules/sfp_callername.py
index d8768db9..6fa80ad2 100644
--- a/modules/sfp_callername.py
+++ b/modules/sfp_callername.py
@@ -13,7 +13,7 @@
import re
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_callername(SpiderFootPlugin):
"""CallerName:Footprint,Investigate,Passive:Real World::Lookup US phone number location and reputation information."""
@@ -21,9 +21,9 @@ class sfp_callername(SpiderFootPlugin):
meta = {
'name': "CallerName",
'summary': "Lookup US phone number location and reputation information.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Real World" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Real World"],
'dataSource': {
'website': "http://callername.com/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_censys.py b/modules/sfp_censys.py
index 4a5bcb06..4269f379 100644
--- a/modules/sfp_censys.py
+++ b/modules/sfp_censys.py
@@ -15,7 +15,7 @@ import base64
from datetime import datetime
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_censys(SpiderFootPlugin):
"""Censys:Investigate,Passive:Search Engines:apikey:Obtain information from Censys.io"""
@@ -23,9 +23,9 @@ class sfp_censys(SpiderFootPlugin):
meta = {
'name': "Censys",
'summary': "Obtain information from Censys.io",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ]
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Search Engines"]
}
# Default options
@@ -91,7 +91,7 @@ class sfp_censys(SpiderFootPlugin):
# API rate limit: 0.4 actions/second (120.0 per 5 minute interval)
time.sleep(self.opts['delay'])
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("Censys.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)
self.errorState = True
return None
@@ -103,7 +103,7 @@ class sfp_censys(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from Censys.io.", False)
+ self.sf.error(f"Error processing JSON response from Censys.io: {e}", False)
return None
#print(str(info))
diff --git a/modules/sfp_cinsscore.py b/modules/sfp_cinsscore.py
index 746f2fed..f356abbc 100644
--- a/modules/sfp_cinsscore.py
+++ b/modules/sfp_cinsscore.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress, IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_cinsscore(SpiderFootPlugin):
"""CINS Army List:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is malicious according to cinsscore.com's Army List."""
@@ -19,9 +19,9 @@ class sfp_cinsscore(SpiderFootPlugin):
meta = {
'name': "CINS Army List",
'summary': "Check if a netblock or IP address is malicious according to cinsscore.com's Army List.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ]
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"]
}
# Default options
diff --git a/modules/sfp_circllu.py b/modules/sfp_circllu.py
index 65379d97..e50dff01 100644
--- a/modules/sfp_circllu.py
+++ b/modules/sfp_circllu.py
@@ -15,7 +15,7 @@ import base64
import re
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_circllu(SpiderFootPlugin):
"""CIRCL.LU:Investigate,Passive:Reputation Systems:apikey:Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases."""
@@ -23,9 +23,9 @@ class sfp_circllu(SpiderFootPlugin):
meta = {
'name': "CIRCL.LU",
'summary': "Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.circl.lu/",
'model': "FREE_AUTH_UNLIMITED",
@@ -98,8 +98,6 @@ class sfp_circllu(SpiderFootPlugin):
return ["IP_ADDRESS", "SSL_CERTIFICATE_ISSUED", "CO_HOSTED_SITE"]
def query(self, qry, qtype):
- ret = None
-
if self.errorState:
return None
@@ -118,7 +116,7 @@ class sfp_circllu(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=30,
useragent="SpiderFoot", headers=headers)
- if res['code'] not in [ "200", "201" ]:
+ if res['code'] not in ["200", "201"]:
self.sf.error("CIRCL.LU access seems to have been rejected or you have exceeded usage limits.", False)
self.errorState = True
return None
@@ -158,7 +156,7 @@ class sfp_circllu(SpiderFootPlugin):
else:
self.results[eventData] = True
- if eventName in [ 'IP_ADDRESS', 'NETBLOCK_OWNER' ]:
+ if eventName in ['IP_ADDRESS', 'NETBLOCK_OWNER']:
# CIRCL.LU limit the maximum subnet size to 23
# http://circl.lu/services/passive-ssl/
if "/" in eventData:
@@ -185,7 +183,7 @@ class sfp_circllu(SpiderFootPlugin):
ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event)
self.notifyListeners(ipe)
for crt in j[ip]['subjects']:
- r = re.findall(".*[\"\'](.+CN=([a-zA-Z0-9\-\*\.])+)[\"\'].*",
+ r = re.findall(r".*[\"\'](.+CN=([a-zA-Z0-9\-\*\.])+)[\"\'].*",
str(j[ip]['subjects'][crt]), re.IGNORECASE)
if r:
e = SpiderFootEvent("SSL_CERTIFICATE_ISSUED", r[0][0], self.__name__, ipe)
@@ -193,7 +191,7 @@ class sfp_circllu(SpiderFootPlugin):
except BaseException as e:
self.sf.error("Invalid response returned from CIRCL.LU: " + str(e), False)
- if eventName in [ 'IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME' ]:
+ if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']:
ret = self.query(eventData, "PDNS")
if not ret:
self.sf.info("No CIRCL.LU passive DNS data found for " + eventData)
@@ -222,7 +220,7 @@ class sfp_circllu(SpiderFootPlugin):
# We found a co-host
cohosts.append(rec['rrname'])
- if eventName in [ "INTERNET_NAME", "DOMAIN_NAME" ]:
+ if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]:
# Record could be an A/CNAME of this entity, or something pointing to it
if rec['rdata'] == eventData:
if not self.getTarget().matches(rec['rrname']):
diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.py
index 02c4a152..26cbb35a 100644
--- a/modules/sfp_citadel.py
+++ b/modules/sfp_citadel.py
@@ -12,7 +12,7 @@
import json
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_citadel(SpiderFootPlugin):
"""Leak-Lookup:Footprint,Investigate,Passive:Leaks, Dumps and Breaches:apikey:Searches Leak-Lookup.com's database of breaches."""
@@ -20,9 +20,9 @@ class sfp_citadel(SpiderFootPlugin):
meta = {
'name': "Leak-Lookup",
'summary': "Searches Leak-Lookup.com's database of breaches.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://leak-lookup.com/",
'model': "FREE_AUTH_UNLIMITED",
@@ -126,7 +126,7 @@ class sfp_citadel(SpiderFootPlugin):
# Don't look up stuff twice
if eventData in self.results:
- self.sf.debug("Skipping " + eventData + " as already searched.")
+ self.sf.debug(f"Skipping {eventData}, already checked.")
return None
self.results[eventData] = True
@@ -140,7 +140,7 @@ class sfp_citadel(SpiderFootPlugin):
message = data.get('message')
if error == 'true':
- self.sf.error("Error encountered processing {}: {}".format( eventData, message ), False)
+ self.sf.error(f"Error encountered processing {eventData}: {message}")
if "MISSING API" in message:
self.errorState = True
return None
@@ -150,8 +150,8 @@ class sfp_citadel(SpiderFootPlugin):
return None
for site in message:
- self.sf.info("Found Leak-Lookup entry for {}: {}".format( eventData, site ) )
- evt = SpiderFootEvent( "EMAILADDR_COMPROMISED", "{} [{}]".format( eventData, site ), self.__name__, event )
+ self.sf.info(f"Found Leak-Lookup entry for {eventData}: {site}")
+ evt = SpiderFootEvent("EMAILADDR_COMPROMISED", f"{eventData} [{site}]", self.__name__, event)
self.notifyListeners(evt)
# End of sfp_citadel class
diff --git a/modules/sfp_cleanbrowsing.py b/modules/sfp_cleanbrowsing.py
index bd25d745..93c52c2b 100644
--- a/modules/sfp_cleanbrowsing.py
+++ b/modules/sfp_cleanbrowsing.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_cleanbrowsing(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_cleanbrowsing(SpiderFootPlugin):
meta = {
'name': "Cleanbrowsing.org",
'summary': "Check if a host would be blocked by Cleanbrowsing.org DNS",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://cleanbrowsing.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -76,13 +76,13 @@ class sfp_cleanbrowsing(SpiderFootPlugin):
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "185.228.168.168", "185.228.168.169" ]
+ res.nameservers = ["185.228.168.168", "185.228.168.169"]
try:
addrs = res.query(qaddr)
self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -108,7 +108,7 @@ class sfp_cleanbrowsing(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
+ except BaseException:
return None
if resolved:
diff --git a/modules/sfp_cleantalk.py b/modules/sfp_cleantalk.py
index 1fda22e7..0b1aaa34 100644
--- a/modules/sfp_cleantalk.py
+++ b/modules/sfp_cleantalk.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress, IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_cleantalk(SpiderFootPlugin):
"""CleanTalk Spam List:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is on CleanTalk.org's spam IP list."""
@@ -19,9 +19,9 @@ class sfp_cleantalk(SpiderFootPlugin):
meta = {
'name': "CleanTalk Spam List",
'summary': "Check if a netblock or IP address is on CleanTalk.org's spam IP list.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://cleantalk.org",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_clearbit.py b/modules/sfp_clearbit.py
index b2f582b4..e71b2e3f 100644
--- a/modules/sfp_clearbit.py
+++ b/modules/sfp_clearbit.py
@@ -11,7 +11,7 @@
import json
import base64
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_clearbit(SpiderFootPlugin):
"""Clearbit:Footprint,Investigate,Passive:Search Engines:apikey:Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com."""
@@ -19,9 +19,9 @@ class sfp_clearbit(SpiderFootPlugin):
meta = {
'name': "Clearbit",
'summary': "Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://clearbit.com/",
'model': "FREE_AUTH_LIMITED",
@@ -77,12 +77,12 @@ class sfp_clearbit(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "EMAILADDR" ]
+ return ["EMAILADDR"]
# What events this module produces
def producedEvents(self):
- return [ "RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS",
- "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC" ]
+ return ["RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS",
+ "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC"]
def query(self, t):
ret = None
@@ -97,19 +97,16 @@ class sfp_clearbit(SpiderFootPlugin):
'Authorization': "Basic " + token.decode('utf-8')
}
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent="SpiderFoot", headers=headers)
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers)
if res['code'] != "200":
- self.sf.error("Return code indicates no results or potential API key failure or exceeded limits.",
- False)
+ self.sf.error("Return code indicates no results or potential API key failure or exceeded limits.", False)
return None
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from clearbit.io: " + \
- str(e), False)
+ self.sf.error(f"Error processing JSON response from clearbit.io: {e}", False)
return None
return ret
diff --git a/modules/sfp_cloudflaredns.py b/modules/sfp_cloudflaredns.py
index 35ddf762..f3604c64 100644
--- a/modules/sfp_cloudflaredns.py
+++ b/modules/sfp_cloudflaredns.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_cloudflaredns(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_cloudflaredns(SpiderFootPlugin):
meta = {
'name': "CloudFlare Malware DNS",
'summary': "Check if a host would be blocked by CloudFlare Malware-blocking DNS",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.cloudflare.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -69,13 +69,13 @@ class sfp_cloudflaredns(SpiderFootPlugin):
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "1.1.1.2", "1.0.0.2" ]
+ res.nameservers = ["1.1.1.2", "1.0.0.2"]
try:
addrs = res.query(qaddr)
self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -106,7 +106,7 @@ class sfp_cloudflaredns(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
+ except BaseException:
return None
if resolved:
diff --git a/modules/sfp_coinblocker.py b/modules/sfp_coinblocker.py
index 7c6a8c81..4a7744b7 100644
--- a/modules/sfp_coinblocker.py
+++ b/modules/sfp_coinblocker.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'CoinBlocker IP List': {
@@ -36,9 +36,9 @@ class sfp_coinblocker(SpiderFootPlugin):
meta = {
'name': "CoinBlocker Lists",
'summary': "Check if a host/domain or IP appears on CoinBlocker lists.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://zerodot1.gitlab.io/CoinBlockerListsWeb/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -138,8 +138,7 @@ class sfp_coinblocker(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_commoncrawl.py b/modules/sfp_commoncrawl.py
index b941bc15..cf5edac9 100644
--- a/modules/sfp_commoncrawl.py
+++ b/modules/sfp_commoncrawl.py
@@ -14,7 +14,7 @@
import re
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_commoncrawl(SpiderFootPlugin):
@@ -23,9 +23,9 @@ class sfp_commoncrawl(SpiderFootPlugin):
meta = {
'name': "CommonCrawl",
'summary': "Searches for URLs found through CommonCrawl.org.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "http://commoncrawl.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -74,7 +74,7 @@ class sfp_commoncrawl(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=60,
useragent="SpiderFoot")
- if res['code'] in [ "400", "401", "402", "403", "404" ]:
+ if res['code'] in ["400", "401", "402", "403", "404"]:
self.sf.error("CommonCrawl search doesn't seem to be available.", False)
self.errorState = True
return None
@@ -93,7 +93,7 @@ class sfp_commoncrawl(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=60,
useragent="SpiderFoot")
- if res['code'] in [ "400", "401", "402", "403", "404" ]:
+ if res['code'] in ["400", "401", "402", "403", "404"]:
self.sf.error("CommonCrawl index collection doesn't seem to be available.", False)
self.errorState = True
return list()
@@ -103,8 +103,7 @@ class sfp_commoncrawl(SpiderFootPlugin):
self.errorState = True
return list()
- indexes = re.findall(".*(CC-MAIN-\d+-\d+).*", res['content'])
- highest = 0
+ indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", res['content'])
indexlist = dict()
for m in indexes:
ms = m.replace("CC-MAIN-", "").replace("-", "")
diff --git a/modules/sfp_comodo.py b/modules/sfp_comodo.py
index 94704fc6..008e7b1b 100644
--- a/modules/sfp_comodo.py
+++ b/modules/sfp_comodo.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_comodo(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_comodo(SpiderFootPlugin):
meta = {
'name': "Comodo",
'summary': "Check if a host would be blocked by Comodo DNS",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.comodo.com/secure-dns/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -72,13 +72,13 @@ class sfp_comodo(SpiderFootPlugin):
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "8.26.56.26", "8.20.247.20" ]
+ res.nameservers = ["8.26.56.26", "8.20.247.20"]
try:
addrs = res.query(qaddr)
self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -104,7 +104,7 @@ class sfp_comodo(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
+ except BaseException:
return None
if resolved:
diff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.py
index e667b2db..29f58829 100644
--- a/modules/sfp_cookie.py
+++ b/modules/sfp_cookie.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_cookie(SpiderFootPlugin):
"""Cookie Extractor:Footprint,Investigate,Passive:Content Analysis::Extract Cookies from HTTP headers."""
@@ -19,9 +19,9 @@ class sfp_cookie(SpiderFootPlugin):
meta = {
'name': "Cookie Extractor",
'summary': "Extract Cookies from HTTP headers.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -69,7 +69,7 @@ class sfp_cookie(SpiderFootPlugin):
jdata = json.loads(eventData)
if jdata == None:
return None
- except BaseException as e:
+ except BaseException:
self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
return None
diff --git a/modules/sfp_countryname.py b/modules/sfp_countryname.py
index 1ba926f2..f5275bd4 100644
--- a/modules/sfp_countryname.py
+++ b/modules/sfp_countryname.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import phonenumbers
from phonenumbers.phonenumberutil import region_code_for_country_code
@@ -22,28 +22,28 @@ class sfp_countryname(SpiderFootPlugin):
meta = {
'name': "Country Name Extractor",
'summary': "Identify country names in any obtained data.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
opts = {
# options specific to this module
- 'cohosted' : True,
- 'affiliate' : True,
- 'noncountrytld' : True,
- 'noncountrytlddefault' : "United States",
- 'similardomain' : False,
+ 'cohosted': True,
+ 'affiliate': True,
+ 'noncountrytld': True,
+ 'noncountrytlddefault': "United States",
+ 'similardomain': False,
}
# Option descriptions
optdescs = {
- 'cohosted' : "Obtain country name from co-hosted sites",
- 'affiliate' : "Obtain country name from affiliate sites",
- 'noncountrytld' : "Parse TLDs not associated with any country as default country domains",
- 'noncountrytlddefault' : "Default country name for TLDs not associated with any country(.com, .net, .org, .gov, .mil)",
- 'similardomain' : "Obtain country name from similar domains"
+ 'cohosted': "Obtain country name from co-hosted sites",
+ 'affiliate': "Obtain country name from affiliate sites",
+ 'noncountrytld': "Parse TLDs not associated with any country as default country domains",
+ 'noncountrytlddefault': "Default country name for TLDs not associated with any country(.com, .net, .org, .gov, .mil)",
+ 'similardomain': "Obtain country name from similar domains"
}
results = None
@@ -62,93 +62,263 @@ class sfp_countryname(SpiderFootPlugin):
# Dictionary of country codes and country names
abbvCountryCodes = {
- "AF" : "Afghanistan", "AX" : "Aland Islands", "AL" : "Albania",
- "DZ" : "Algeria", "AS" : "American Samoa", "AD" : "Andorra",
- "AO" : "Angola", "AI" : "Anguilla", "AQ" : "Antarctica",
- "AG" : "Antigua and Barbuda", "AR" : "Argentina", "AM" : "Armenia",
- "AW" : "Aruba", "AU" : "Australia", "AT" : "Austria",
- "AZ" : "Azerbaijan", "BS" : "Bahamas", "BH" : "Bahrain",
- "BD" : "Bangladesh", "BB" : "Barbados", "BY" : "Belarus",
- "BE" : "Belgium", "BZ" : "Belize", "BJ" : "Benin",
- "BM" : "Bermuda", "BT" : "Bhutan", "BO" : "Bolivia",
- "BQ" : "Bonaire, Saint Eustatius and Saba", "BA" : "Bosnia and Herzegovina", "BW" : "Botswana",
- "BV" : "Bouvet Island", "BR" : "Brazil", "IO" : "British Indian Ocean Territory",
- "VG" : "British Virgin Islands", "BN" : "Brunei", "BG" : "Bulgaria",
- "BF" : "Burkina Faso", "BI" : "Burundi", "KH" : "Cambodia",
- "CM" : "Cameroon", "CA" : "Canada", "CV" : "Cape Verde",
- "KY" : "Cayman Islands", "CF" : "Central African Republic", "TD" : "Chad",
- "CL" : "Chile", "CN" : "China", "CX" : "Christmas Island",
- "CC" : "Cocos Islands", "CO" : "Colombia", "KM" : "Comoros",
- "CK" : "Cook Islands", "CR" : "Costa Rica", "HR" : "Croatia",
- "CU" : "Cuba", "CW" : "Curacao", "CY" : "Cyprus",
- "CZ" : "Czech Republic", "CD" : "Democratic Republic of the Congo", "DK" : "Denmark",
- "DJ" : "Djibouti", "DM" : "Dominica", "DO" : "Dominican Republic",
- "TL" : "East Timor", "EC" : "Ecuador", "EG" : "Egypt",
- "SV" : "El Salvador", "GQ" : "Equatorial Guinea", "ER" : "Eritrea",
- "EE" : "Estonia", "ET" : "Ethiopia", "FK" : "Falkland Islands",
- "FO" : "Faroe Islands", "FJ" : "Fiji", "FI" : "Finland",
- "FR" : "France", "GF" : "French Guiana", "PF" : "French Polynesia",
- "TF" : "French Southern Territories", "GA" : "Gabon", "GM" : "Gambia",
- "GE" : "Georgia", "DE" : "Germany", "GH" : "Ghana",
- "GI" : "Gibraltar", "GR" : "Greece", "GL" : "Greenland",
- "GD" : "Grenada", "GP" : "Guadeloupe", "GU" : "Guam",
- "GT" : "Guatemala", "GG" : "Guernsey", "GN" : "Guinea",
- "GW" : "Guinea-Bissau", "GY" : "Guyana", "HT" : "Haiti",
- "HM" : "Heard Island and McDonald Islands", "HN" : "Honduras", "HK" : "Hong Kong",
- "HU" : "Hungary", "IS" : "Iceland", "IN" : "India",
- "ID" : "Indonesia", "IR" : "Iran", "IQ" : "Iraq",
- "IE" : "Ireland", "IM" : "Isle of Man", "IL" : "Israel",
- "IT" : "Italy", "CI" : "Ivory Coast", "JM" : "Jamaica",
- "JP" : "Japan", "JE" : "Jersey", "JO" : "Jordan",
- "KZ" : "Kazakhstan", "KE" : "Kenya", "KI" : "Kiribati",
- "XK" : "Kosovo", "KW" : "Kuwait", "KG" : "Kyrgyzstan",
- "LA" : "Laos", "LV" : "Latvia", "LB" : "Lebanon",
- "LS" : "Lesotho", "LR" : "Liberia", "LY" : "Libya",
- "LI" : "Liechtenstein", "LT" : "Lithuania", "LU" : "Luxembourg",
- "MO" : "Macao", "MK" : "Macedonia", "MG" : "Madagascar",
- "MW" : "Malawi", "MY" : "Malaysia", "MV" : "Maldives",
- "ML" : "Mali", "MT" : "Malta", "MH" : "Marshall Islands",
- "MQ" : "Martinique", "MR" : "Mauritania", "MU" : "Mauritius",
- "YT" : "Mayotte", "MX" : "Mexico", "FM" : "Micronesia",
- "MD" : "Moldova", "MC" : "Monaco", "MN" : "Mongolia",
- "ME" : "Montenegro", "MS" : "Montserrat", "MA" : "Morocco",
- "MZ" : "Mozambique", "MM" : "Myanmar", "NA" : "Namibia",
- "NR" : "Nauru", "NP" : "Nepal", "NL" : "Netherlands",
- "AN" : "Netherlands Antilles", "NC" : "New Caledonia", "NZ" : "New Zealand",
- "NI" : "Nicaragua", "NE" : "Niger", "NG" : "Nigeria",
- "NU" : "Niue", "NF" : "Norfolk Island", "KP" : "North Korea",
- "MP" : "Northern Mariana Islands", "NO" : "Norway", "OM" : "Oman",
- "PK" : "Pakistan", "PW" : "Palau", "PS" : "Palestinian Territory",
- "PA" : "Panama", "PG" : "Papua New Guinea", "PY" : "Paraguay",
- "PE" : "Peru", "PH" : "Philippines", "PN" : "Pitcairn",
- "PL" : "Poland", "PT" : "Portugal", "PR" : "Puerto Rico",
- "QA" : "Qatar", "CG" : "Republic of the Congo", "RE" : "Reunion",
- "RO" : "Romania", "RU" : "Russia", "RW" : "Rwanda",
- "BL" : "Saint Barthelemy", "SH" : "Saint Helena", "KN" : "Saint Kitts and Nevis",
- "LC" : "Saint Lucia", "MF" : "Saint Martin", "PM" : "Saint Pierre and Miquelon",
- "VC" : "Saint Vincent and the Grenadines", "WS" : "Samoa", "SM" : "San Marino",
- "ST" : "Sao Tome and Principe", "SA" : "Saudi Arabia", "SN" : "Senegal",
- "RS" : "Serbia", "CS" : "Serbia and Montenegro", "SC" : "Seychelles",
- "SL" : "Sierra Leone", "SG" : "Singapore", "SX" : "Sint Maarten",
- "SK" : "Slovakia", "SI" : "Slovenia", "SB" : "Solomon Islands",
- "SO" : "Somalia", "ZA" : "South Africa", "GS" : "South Georgia and the South Sandwich Islands",
- "KR" : "South Korea", "SS" : "South Sudan", "ES" : "Spain",
- "LK" : "Sri Lanka", "SD" : "Sudan", "SR" : "Suriname",
- "SJ" : "Svalbard and Jan Mayen", "SZ" : "Swaziland", "SE" : "Sweden",
- "CH" : "Switzerland", "SY" : "Syria", "TW" : "Taiwan",
- "TJ" : "Tajikistan", "TZ" : "Tanzania", "TH" : "Thailand",
- "TG" : "Togo", "TK" : "Tokelau", "TO" : "Tonga",
- "TT" : "Trinidad and Tobago", "TN" : "Tunisia", "TR" : "Turkey",
- "TM" : "Turkmenistan", "TC" : "Turks and Caicos Islands", "TV" : "Tuvalu",
- "VI" : "U.S. Virgin Islands", "UG" : "Uganda", "UA" : "Ukraine",
- "AE" : "United Arab Emirates", "GB" : "United Kingdom", "US" : "United States",
- "UM" : "United States Minor Outlying Islands", "UY" : "Uruguay", "UZ" : "Uzbekistan",
- "VU" : "Vanuatu", "VA" : "Vatican", "VE" : "Venezuela",
- "VN" : "Vietnam", "WF" : "Wallis and Futuna", "EH" : "Western Sahara",
- "YE" : "Yemen", "ZM" : "Zambia", "ZW" : "Zimbabwe",
+ "AF": "Afghanistan",
+ "AX": "Aland Islands",
+ "AL": "Albania",
+ "DZ": "Algeria",
+ "AS": "American Samoa",
+ "AD": "Andorra",
+ "AO": "Angola",
+ "AI": "Anguilla",
+ "AQ": "Antarctica",
+ "AG": "Antigua and Barbuda",
+ "AR": "Argentina",
+ "AM": "Armenia",
+ "AW": "Aruba",
+ "AU": "Australia",
+ "AT": "Austria",
+ "AZ": "Azerbaijan",
+ "BS": "Bahamas",
+ "BH": "Bahrain",
+ "BD": "Bangladesh",
+ "BB": "Barbados",
+ "BY": "Belarus",
+ "BE": "Belgium",
+ "BZ": "Belize",
+ "BJ": "Benin",
+ "BM": "Bermuda",
+ "BT": "Bhutan",
+ "BO": "Bolivia",
+ "BQ": "Bonaire, Saint Eustatius and Saba",
+ "BA": "Bosnia and Herzegovina",
+ "BW": "Botswana",
+ "BV": "Bouvet Island",
+ "BR": "Brazil",
+ "IO": "British Indian Ocean Territory",
+ "VG": "British Virgin Islands",
+ "BN": "Brunei",
+ "BG": "Bulgaria",
+ "BF": "Burkina Faso",
+ "BI": "Burundi",
+ "KH": "Cambodia",
+ "CM": "Cameroon",
+ "CA": "Canada",
+ "CV": "Cape Verde",
+ "KY": "Cayman Islands",
+ "CF": "Central African Republic",
+ "TD": "Chad",
+ "CL": "Chile",
+ "CN": "China",
+ "CX": "Christmas Island",
+ "CC": "Cocos Islands",
+ "CO": "Colombia",
+ "KM": "Comoros",
+ "CK": "Cook Islands",
+ "CR": "Costa Rica",
+ "HR": "Croatia",
+ "CU": "Cuba",
+ "CW": "Curacao",
+ "CY": "Cyprus",
+ "CZ": "Czech Republic",
+ "CD": "Democratic Republic of the Congo",
+ "DK": "Denmark",
+ "DJ": "Djibouti",
+ "DM": "Dominica",
+ "DO": "Dominican Republic",
+ "TL": "East Timor",
+ "EC": "Ecuador",
+ "EG": "Egypt",
+ "SV": "El Salvador",
+ "GQ": "Equatorial Guinea",
+ "ER": "Eritrea",
+ "EE": "Estonia",
+ "ET": "Ethiopia",
+ "FK": "Falkland Islands",
+ "FO": "Faroe Islands",
+ "FJ": "Fiji",
+ "FI": "Finland",
+ "FR": "France",
+ "GF": "French Guiana",
+ "PF": "French Polynesia",
+ "TF": "French Southern Territories",
+ "GA": "Gabon",
+ "GM": "Gambia",
+ "GE": "Georgia",
+ "DE": "Germany",
+ "GH": "Ghana",
+ "GI": "Gibraltar",
+ "GR": "Greece",
+ "GL": "Greenland",
+ "GD": "Grenada",
+ "GP": "Guadeloupe",
+ "GU": "Guam",
+ "GT": "Guatemala",
+ "GG": "Guernsey",
+ "GN": "Guinea",
+ "GW": "Guinea-Bissau",
+ "GY": "Guyana",
+ "HT": "Haiti",
+ "HM": "Heard Island and McDonald Islands",
+ "HN": "Honduras",
+ "HK": "Hong Kong",
+ "HU": "Hungary",
+ "IS": "Iceland",
+ "IN": "India",
+ "ID": "Indonesia",
+ "IR": "Iran",
+ "IQ": "Iraq",
+ "IE": "Ireland",
+ "IM": "Isle of Man",
+ "IL": "Israel",
+ "IT": "Italy",
+ "CI": "Ivory Coast",
+ "JM": "Jamaica",
+ "JP": "Japan",
+ "JE": "Jersey",
+ "JO": "Jordan",
+ "KZ": "Kazakhstan",
+ "KE": "Kenya",
+ "KI": "Kiribati",
+ "XK": "Kosovo",
+ "KW": "Kuwait",
+ "KG": "Kyrgyzstan",
+ "LA": "Laos",
+ "LV": "Latvia",
+ "LB": "Lebanon",
+ "LS": "Lesotho",
+ "LR": "Liberia",
+ "LY": "Libya",
+ "LI": "Liechtenstein",
+ "LT": "Lithuania",
+ "LU": "Luxembourg",
+ "MO": "Macao",
+ "MK": "Macedonia",
+ "MG": "Madagascar",
+ "MW": "Malawi",
+ "MY": "Malaysia",
+ "MV": "Maldives",
+ "ML": "Mali",
+ "MT": "Malta",
+ "MH": "Marshall Islands",
+ "MQ": "Martinique",
+ "MR": "Mauritania",
+ "MU": "Mauritius",
+ "YT": "Mayotte",
+ "MX": "Mexico",
+ "FM": "Micronesia",
+ "MD": "Moldova",
+ "MC": "Monaco",
+ "MN": "Mongolia",
+ "ME": "Montenegro",
+ "MS": "Montserrat",
+ "MA": "Morocco",
+ "MZ": "Mozambique",
+ "MM": "Myanmar",
+ "NA": "Namibia",
+ "NR": "Nauru",
+ "NP": "Nepal",
+ "NL": "Netherlands",
+ "AN": "Netherlands Antilles",
+ "NC": "New Caledonia",
+ "NZ": "New Zealand",
+ "NI": "Nicaragua",
+ "NE": "Niger",
+ "NG": "Nigeria",
+ "NU": "Niue",
+ "NF": "Norfolk Island",
+ "KP": "North Korea",
+ "MP": "Northern Mariana Islands",
+ "NO": "Norway",
+ "OM": "Oman",
+ "PK": "Pakistan",
+ "PW": "Palau",
+ "PS": "Palestinian Territory",
+ "PA": "Panama",
+ "PG": "Papua New Guinea",
+ "PY": "Paraguay",
+ "PE": "Peru",
+ "PH": "Philippines",
+ "PN": "Pitcairn",
+ "PL": "Poland",
+ "PT": "Portugal",
+ "PR": "Puerto Rico",
+ "QA": "Qatar",
+ "CG": "Republic of the Congo",
+ "RE": "Reunion",
+ "RO": "Romania",
+ "RU": "Russia",
+ "RW": "Rwanda",
+ "BL": "Saint Barthelemy",
+ "SH": "Saint Helena",
+ "KN": "Saint Kitts and Nevis",
+ "LC": "Saint Lucia",
+ "MF": "Saint Martin",
+ "PM": "Saint Pierre and Miquelon",
+ "VC": "Saint Vincent and the Grenadines",
+ "WS": "Samoa",
+ "SM": "San Marino",
+ "ST": "Sao Tome and Principe",
+ "SA": "Saudi Arabia",
+ "SN": "Senegal",
+ "RS": "Serbia",
+ "CS": "Serbia and Montenegro",
+ "SC": "Seychelles",
+ "SL": "Sierra Leone",
+ "SG": "Singapore",
+ "SX": "Sint Maarten",
+ "SK": "Slovakia",
+ "SI": "Slovenia",
+ "SB": "Solomon Islands",
+ "SO": "Somalia",
+ "ZA": "South Africa",
+ "GS": "South Georgia and the South Sandwich Islands",
+ "KR": "South Korea",
+ "SS": "South Sudan",
+ "ES": "Spain",
+ "LK": "Sri Lanka",
+ "SD": "Sudan",
+ "SR": "Suriname",
+ "SJ": "Svalbard and Jan Mayen",
+ "SZ": "Swaziland",
+ "SE": "Sweden",
+ "CH": "Switzerland",
+ "SY": "Syria",
+ "TW": "Taiwan",
+ "TJ": "Tajikistan",
+ "TZ": "Tanzania",
+ "TH": "Thailand",
+ "TG": "Togo",
+ "TK": "Tokelau",
+ "TO": "Tonga",
+ "TT": "Trinidad and Tobago",
+ "TN": "Tunisia",
+ "TR": "Turkey",
+ "TM": "Turkmenistan",
+ "TC": "Turks and Caicos Islands",
+ "TV": "Tuvalu",
+ "VI": "U.S. Virgin Islands",
+ "UG": "Uganda",
+ "UA": "Ukraine",
+ "AE": "United Arab Emirates",
+ "GB": "United Kingdom",
+ "US": "United States",
+ "UM": "United States Minor Outlying Islands",
+ "UY": "Uruguay",
+ "UZ": "Uzbekistan",
+ "VU": "Vanuatu",
+ "VA": "Vatican",
+ "VE": "Venezuela",
+ "VN": "Vietnam",
+ "WF": "Wallis and Futuna",
+ "EH": "Western Sahara",
+ "YE": "Yemen",
+ "ZM": "Zambia",
+ "ZW": "Zimbabwe",
# Below are not country codes but recognized as TLDs
- "AC" : "Ascension Island", "EU" : "European Union", "SU" : "Soviet Union",
- "UK" : "United Kingdom"
+ "AC": "Ascension Island",
+ "EU": "European Union",
+ "SU": "Soviet Union",
+ "UK": "United Kingdom"
}
return abbvCountryCodes
@@ -241,7 +411,7 @@ class sfp_countryname(SpiderFootPlugin):
# Look for country name in source data
# Spaces are not included since New Jersey and others
# will get interpreted as Jersey, etc.
- matchCountries = re.findall(r"[,'\"\:\=\[\(\[\n\t\r\.] ?" + countryName + "[,'\"\:\=\[\(\[\n\t\r\.]", srcData, re.IGNORECASE)
+ matchCountries = re.findall(r"[,'\"\:\=\[\(\[\n\t\r\.] ?" + countryName + r"[,'\"\:\=\[\(\[\n\t\r\.]", srcData, re.IGNORECASE)
if len(matchCountries) > 0:
# Get country name from first index of list
@@ -306,7 +476,7 @@ class sfp_countryname(SpiderFootPlugin):
# Process the event data based on incoming event type
if eventName == "PHONE_NUMBER":
countryNames.append(self.detectCountryFromPhone(eventData))
- elif eventName == "DOMAIN_NAME" or (eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]) or(eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]) or (eventName == "SIMILARDOMAIN" and self.opts["similardomain"]):
+ elif eventName == "DOMAIN_NAME" or (eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]) or (eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]) or (eventName == "SIMILARDOMAIN" and self.opts["similardomain"]):
countryNames.append(self.detectCountryFromTLD(eventData))
elif eventName == "IBAN_NUMBER":
countryNames.append(self.detectCountryFromIBAN(eventData))
diff --git a/modules/sfp_creditcard.py b/modules/sfp_creditcard.py
index 16220ab0..75b6dfc7 100644
--- a/modules/sfp_creditcard.py
+++ b/modules/sfp_creditcard.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_creditcard(SpiderFootPlugin):
"""Credit Card Number Extractor:Footprint,Investigate,Passive:Content Analysis::Identify Credit Card Numbers in any data"""
@@ -19,9 +19,9 @@ class sfp_creditcard(SpiderFootPlugin):
meta = {
'name': "Credit Card Number Extractor",
'summary': "Identify Credit Card Numbers in any data",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options.
@@ -55,7 +55,7 @@ class sfp_creditcard(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "DARKNET_MENTION_CONTENT", "LEAKSITE_CONTENT" ]
+ return ["DARKNET_MENTION_CONTENT", "LEAKSITE_CONTENT"]
# What events this module produces
def producedEvents(self):
diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.py
index b7717724..897a9076 100644
--- a/modules/sfp_crossref.py
+++ b/modules/sfp_crossref.py
@@ -15,7 +15,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_crossref(SpiderFootPlugin):
"""Cross-Referencer:Footprint:Crawling and Scanning::Identify whether other domains are associated ('Affiliates') of the target."""
@@ -23,9 +23,9 @@ class sfp_crossref(SpiderFootPlugin):
meta = {
'name': "Cross-Referencer",
'summary': "Identify whether other domains are associated ('Affiliates') of the target.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint" ],
- 'categories': [ "Crawling and Scanning" ]
+ 'flags': [""],
+ 'useCases': ["Footprint"],
+ 'categories': ["Crawling and Scanning"]
}
# Default options
@@ -97,7 +97,7 @@ class sfp_crossref(SpiderFootPlugin):
matched = False
for name in self.getTarget().getNames():
# Search for mentions of our host/domain in the external site's data
- pat = re.compile("([\.\'\/\"\ ]" + name + "[\.\'\/\"\ ])", re.IGNORECASE)
+ pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\.\'\/\"\ ])", re.IGNORECASE)
matches = re.findall(pat, res['content'])
if len(matches) > 0:
@@ -122,7 +122,7 @@ class sfp_crossref(SpiderFootPlugin):
verify=False)
if res['content'] is not None:
for name in self.getTarget().getNames():
- pat = re.compile("([\.\'\/\"\ ]" + name + "[\'\/\"\ ])",
+ pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\'\/\"\ ])",
re.IGNORECASE)
matches = re.findall(pat, res['content'])
diff --git a/modules/sfp_crt.py b/modules/sfp_crt.py
index 26576334..4c80ba11 100644
--- a/modules/sfp_crt.py
+++ b/modules/sfp_crt.py
@@ -13,7 +13,7 @@
import json
import urllib.parse
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_crt(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_crt(SpiderFootPlugin):
meta = {
'name': "Certificate Transparency",
'summary': "Gather hostnames from historical certificates in crt.sh.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://crt.sh/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_customfeed.py b/modules/sfp_customfeed.py
index 5bc20887..9bf7c852 100644
--- a/modules/sfp_customfeed.py
+++ b/modules/sfp_customfeed.py
@@ -14,7 +14,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'Custom Threat Data': {
@@ -30,9 +30,9 @@ class sfp_customfeed(SpiderFootPlugin):
meta = {
'name': "Custom Threat Feed",
'summary': "Check if a host/domain, netblock, ASN or IP is malicious according to your custom feed.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ]
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"]
}
# Default options
@@ -112,8 +112,7 @@ class sfp_customfeed(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.py
index fc2c4624..09a122a2 100644
--- a/modules/sfp_cybercrimetracker.py
+++ b/modules/sfp_cybercrimetracker.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'cybercrime-tracker.net Malicious Submissions': {
@@ -31,9 +31,9 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
meta = {
'name': "cybercrime-tracker.net",
'summary': "Check if a host/domain or IP is malicious according to cybercrime-tracker.net.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://cybercrime-tracker.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -122,8 +122,7 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_darksearch.py b/modules/sfp_darksearch.py
index bc81cc91..34d05eea 100644
--- a/modules/sfp_darksearch.py
+++ b/modules/sfp_darksearch.py
@@ -14,7 +14,7 @@
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_darksearch(SpiderFootPlugin):
"""Darksearch:Footprint,Investigate:Search Engines::Search the Darksearch.io Tor search engine for mentions of the target domain."""
@@ -22,9 +22,9 @@ class sfp_darksearch(SpiderFootPlugin):
meta = {
'name': "Darksearch",
'summary': "Search the Darksearch.io Tor search engine for mentions of the target domain.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://darksearch.io/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -181,7 +181,7 @@ class sfp_darksearch(SpiderFootPlugin):
try:
startIndex = res['content'].index(eventData) - 120
endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
+ except BaseException:
self.sf.debug("String not found in content.")
continue
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.py
index af768492..90d25a03 100644
--- a/modules/sfp_digitaloceanspace.py
+++ b/modules/sfp_digitaloceanspace.py
@@ -13,7 +13,7 @@
import threading
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_digitaloceanspace(SpiderFootPlugin):
"""Digital Ocean Space Finder:Footprint,Passive:Crawling and Scanning::Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents."""
@@ -21,9 +21,9 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
meta = {
'name': "Digital Ocean Space Finder",
'summary': "Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Crawling and Scanning" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Crawling and Scanning"],
'dataSource': {
'website': "https://www.digitalocean.com/products/spaces/",
'model': "FREE_NOAUTH_UNLIMITED"
@@ -70,7 +70,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
- if res['code'] not in [ "301", "302", "200" ] and \
+ if res['code'] not in ["301", "302", "200"] and \
(res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
@@ -82,7 +82,6 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
self.s3results[url] = 0
def threadSites(self, siteList):
- ret = list()
self.s3results = dict()
running = True
i = 0
@@ -156,7 +155,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
self.notifyListeners(evt)
return None
- targets = [ eventData.replace('.', '') ]
+ targets = [eventData.replace('.', '')]
kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
if kw:
targets.append(kw)
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.py
index e2fd0194..f61d80d5 100644
--- a/modules/sfp_dnsbrute.py
+++ b/modules/sfp_dnsbrute.py
@@ -14,7 +14,7 @@
import random
import threading
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_dnsbrute(SpiderFootPlugin):
"""DNS Brute-forcer:Footprint,Investigate:DNS::Attempts to identify hostnames through brute-forcing common names and iterations."""
@@ -22,9 +22,9 @@ class sfp_dnsbrute(SpiderFootPlugin):
meta = {
'name': "DNS Brute-forcer",
'summary': "Attempts to identify hostnames through brute-forcing common names and iterations.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["DNS"]
}
# Default options
@@ -98,7 +98,7 @@ class sfp_dnsbrute(SpiderFootPlugin):
if addrs:
with self.lock:
self.hostResults[name] = True
- except BaseException as e:
+ except BaseException:
with self.lock:
self.hostResults[name] = False
@@ -145,7 +145,6 @@ class sfp_dnsbrute(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
eventDataHash = self.sf.hashstring(eventData)
- parentEvent = event
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
diff --git a/modules/sfp_dnscommonsrv.py b/modules/sfp_dnscommonsrv.py
index 56c35ac7..3ed27d9b 100644
--- a/modules/sfp_dnscommonsrv.py
+++ b/modules/sfp_dnscommonsrv.py
@@ -20,9 +20,9 @@ class sfp_dnscommonsrv(SpiderFootPlugin):
meta = {
'name': "DNS Common SRV",
'summary': "Attempts to identify hostnames through common SRV.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["DNS"]
}
# Default options
@@ -33,44 +33,46 @@ class sfp_dnscommonsrv(SpiderFootPlugin):
events = None
- commonsrv = [ # LDAP/Kerberos, used for Active Directory
- # https://technet.microsoft.com/en-us/library/cc961719.aspx
- '_ldap._tcp',
- '_gc._msdcs',
- '_ldap._tcp.pdc._msdcs',
- '_ldap._tcp.gc._msdcs',
- '_kerberos._tcp.dc._msdcs',
- '_kerberos._tcp',
- '_kerberos._udp',
- '_kerberos-master._tcp',
- '_kerberos-master._udp',
- '_kpasswd._tcp',
- '_kpasswd._udp',
- '_ntp._udp',
-
- # SIP
- '_sip._tcp',
- '_sip._udp',
- '_sip._tls',
- '_sips._tcp',
-
- # STUN
- # https://tools.ietf.org/html/rfc5389
- '_stun._tcp',
- '_stun._udp',
- '_stuns._tcp',
-
- # TURN
- # https://tools.ietf.org/html/rfc5928
- '_turn._tcp',
- '_turn._udp',
- '_turns._tcp',
-
- # XMPP
- # http://xmpp.org/rfcs/rfc6120.html
- '_jabber._tcp',
- '_xmpp-client._tcp',
- '_xmpp-server._tcp']
+ commonsrv = [
+ # LDAP/Kerberos, used for Active Directory
+ # https://technet.microsoft.com/en-us/library/cc961719.aspx
+ '_ldap._tcp',
+ '_gc._msdcs',
+ '_ldap._tcp.pdc._msdcs',
+ '_ldap._tcp.gc._msdcs',
+ '_kerberos._tcp.dc._msdcs',
+ '_kerberos._tcp',
+ '_kerberos._udp',
+ '_kerberos-master._tcp',
+ '_kerberos-master._udp',
+ '_kpasswd._tcp',
+ '_kpasswd._udp',
+ '_ntp._udp',
+
+ # SIP
+ '_sip._tcp',
+ '_sip._udp',
+ '_sip._tls',
+ '_sips._tcp',
+
+ # STUN
+ # https://tools.ietf.org/html/rfc5389
+ '_stun._tcp',
+ '_stun._udp',
+ '_stuns._tcp',
+
+ # TURN
+ # https://tools.ietf.org/html/rfc5928
+ '_turn._tcp',
+ '_turn._udp',
+ '_turns._tcp',
+
+ # XMPP
+ # http://xmpp.org/rfcs/rfc6120.html
+ '_jabber._tcp',
+ '_xmpp-client._tcp',
+ '_xmpp-server._tcp'
+ ]
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
@@ -129,7 +131,7 @@ class sfp_dnscommonsrv(SpiderFootPlugin):
try:
answers = res.query(name, 'SRV')
- except BaseException as e:
+ except BaseException:
answers = []
for a in answers:
diff --git a/modules/sfp_dnsgrep.py b/modules/sfp_dnsgrep.py
index 144d2d7a..88403cf1 100644
--- a/modules/sfp_dnsgrep.py
+++ b/modules/sfp_dnsgrep.py
@@ -16,7 +16,7 @@
import json
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_dnsgrep(SpiderFootPlugin):
"""DNSGrep:Footprint,Investigate,Passive:Passive DNS::Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API."""
@@ -24,9 +24,9 @@ class sfp_dnsgrep(SpiderFootPlugin):
meta = {
'name': "DNSGrep",
'summary': "Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "https://opendata.rapid7.com/",
'model': "FREE_AUTH_UNLIMITED",
@@ -99,7 +99,7 @@ class sfp_dnsgrep(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from DNSGrep.", False)
+ self.sf.error(f"Error processing JSON response from DNSGrep: {e}", False)
return None
return data
diff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.py
index 928db313..88b2807c 100644
--- a/modules/sfp_dnsneighbor.py
+++ b/modules/sfp_dnsneighbor.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_dnsneighbor(SpiderFootPlugin):
"""DNS Look-aside:Footprint,Investigate:DNS::Attempt to reverse-resolve the IP addresses next to your target to see if they are related."""
@@ -20,9 +20,9 @@ class sfp_dnsneighbor(SpiderFootPlugin):
meta = {
'name': "DNS Look-aside",
'summary': "Attempt to reverse-resolve the IP addresses next to your target to see if they are related.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["DNS"]
}
# Default options
@@ -79,15 +79,15 @@ class sfp_dnsneighbor(SpiderFootPlugin):
try:
ip = IPAddress(eventData)
- except BaseException as e:
- self.sf.error("Invalid IP address received: " + eventData, False)
+ except BaseException:
+ self.sf.error(f"Invalid IP address received: {eventData}", False)
return None
try:
minip = IPAddress(int(ip) - self.opts['lookasidecount'])
maxip = IPAddress(int(ip) + self.opts['lookasidecount'])
- except BaseException as e:
- self.sf.error("Received an invalid IP address: " + eventData, False)
+ except BaseException:
+ self.sf.error(f"Received an invalid IP address: {eventData}", False)
return None
self.sf.debug("Lookaside max: " + str(maxip) + ", min: " + str(minip))
diff --git a/modules/sfp_dnsraw.py b/modules/sfp_dnsraw.py
index b63795a6..2ea72bc7 100644
--- a/modules/sfp_dnsraw.py
+++ b/modules/sfp_dnsraw.py
@@ -16,7 +16,7 @@ import re
import dns.resolver
import dns.query
import dns.rdatatype
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_dnsraw(SpiderFootPlugin):
"""DNS Raw Records:Footprint,Investigate,Passive:DNS::Retrieves raw DNS records such as MX, TXT and others."""
@@ -24,9 +24,9 @@ class sfp_dnsraw(SpiderFootPlugin):
meta = {
'name': "DNS Raw Records",
'summary': "Retrieves raw DNS records such as MX, TXT and others.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["DNS"]
}
# Default options
diff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.py
index ec4f03e0..76528d1b 100644
--- a/modules/sfp_dnszonexfer.py
+++ b/modules/sfp_dnszonexfer.py
@@ -14,7 +14,7 @@ import re
import dns.query
import dns.zone
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_dnszonexfer(SpiderFootPlugin):
"""DNS Zone Transfer:Footprint,Investigate:DNS::Attempts to perform a full DNS zone transfer."""
@@ -22,9 +22,9 @@ class sfp_dnszonexfer(SpiderFootPlugin):
meta = {
'name': "DNS Zone Transfer",
'summary': "Attempts to perform a full DNS zone transfer.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["DNS"]
}
# Default options
@@ -61,7 +61,6 @@ class sfp_dnszonexfer(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
eventDataHash = self.sf.hashstring(eventData)
- addrs = None
parentEvent = event
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
@@ -114,7 +113,7 @@ class sfp_dnszonexfer(SpiderFootPlugin):
# Try and pull out individual records
for row in ret:
- pat = re.compile("^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)
+ pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)
grps = re.findall(pat, row)
if len(grps) > 0:
for strdata in grps:
diff --git a/modules/sfp_dronebl.py b/modules/sfp_dronebl.py
index 2b3aa3ab..7298e755 100644
--- a/modules/sfp_dronebl.py
+++ b/modules/sfp_dronebl.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_dronebl(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_dronebl(SpiderFootPlugin):
meta = {
'name': "DroneBL",
'summary': "Query the DroneBL database for open relays, open proxies, vulnerable servers, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://dronebl.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -157,7 +157,6 @@ class sfp_dronebl(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
parentEvent = event
- addrlist = list()
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.py
index 84593b91..23ebc6fa 100644
--- a/modules/sfp_duckduckgo.py
+++ b/modules/sfp_duckduckgo.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_duckduckgo(SpiderFootPlugin):
"""DuckDuckGo:Footprint,Investigate,Passive:Search Engines::Query DuckDuckGo's API for descriptive information about your target."""
@@ -19,9 +19,9 @@ class sfp_duckduckgo(SpiderFootPlugin):
meta = {
'name': "DuckDuckGo",
'summary': "Query DuckDuckGo's API for descriptive information about your target.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://duckduckgo.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -72,7 +72,6 @@ class sfp_duckduckgo(SpiderFootPlugin):
def handleEvent(self, event):
eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if self.opts['affiliatedomains'] and "AFFILIATE_" in eventName:
@@ -97,6 +96,7 @@ class sfp_duckduckgo(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except BaseException as e:
+ self.sf.error(f"Error processing JSON response from DuckDuckGo: {e}", False)
return None
if ret['Heading'] == "":
diff --git a/modules/sfp_email.py b/modules/sfp_email.py
index 573d2872..a3d03ac4 100644
--- a/modules/sfp_email.py
+++ b/modules/sfp_email.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_email(SpiderFootPlugin):
"""E-Mail Address Extractor:Footprint,Investigate,Passive:Content Analysis::Identify e-mail addresses in any obtained data."""
@@ -19,8 +19,8 @@ class sfp_email(SpiderFootPlugin):
meta = {
'name': "E-Mail Address Extractor",
'summary': "Identify e-mail addresses in any obtained data.",
- 'useCases': [ "Passive", "Investigate", "Footprint" ],
- 'categories': [ "Content Analysis" ]
+ 'useCases': ["Passive", "Investigate", "Footprint"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_emailcrawlr.py b/modules/sfp_emailcrawlr.py
index d584e4b2..e0db2d5a 100644
--- a/modules/sfp_emailcrawlr.py
+++ b/modules/sfp_emailcrawlr.py
@@ -14,7 +14,7 @@
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_emailcrawlr(SpiderFootPlugin):
"""EmailCrawlr:Footprint,Investigate,Passive:Search Engines:apikey:Search EmailCrawlr for email addresses and phone numbers associated with a domain."""
@@ -22,9 +22,9 @@ class sfp_emailcrawlr(SpiderFootPlugin):
meta = {
'name': "EmailCrawlr",
'summary': "Search EmailCrawlr for email addresses and phone numbers associated with a domain.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://emailcrawlr.com/",
'model': "FREE_AUTH_LIMITED",
@@ -87,8 +87,8 @@ class sfp_emailcrawlr(SpiderFootPlugin):
'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
headers = {
- 'Accept' : "application/json",
- 'x-api-key' : self.opts['api_key']
+ 'Accept': "application/json",
+ 'x-api-key': self.opts['api_key']
}
res = self.sf.fetchUrl(
@@ -135,7 +135,7 @@ class sfp_emailcrawlr(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.py
index 404690d8..4c6c55da 100644
--- a/modules/sfp_emailformat.py
+++ b/modules/sfp_emailformat.py
@@ -13,7 +13,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_emailformat(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_emailformat(SpiderFootPlugin):
meta = {
'name': "EmailFormat",
'summary': "Look up e-mail addresses on email-format.com.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.email-format.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -94,7 +94,7 @@ class sfp_emailformat(SpiderFootPlugin):
continue
# Skip masked emails
- if re.match("^[0-9a-f]{8}\.[0-9]{7}@", email):
+ if re.match(r"^[0-9a-f]{8}\.[0-9]{7}@", email):
self.sf.debug("Skipped address: " + email)
continue
diff --git a/modules/sfp_emailrep.py b/modules/sfp_emailrep.py
index 594fb71c..6dad3d06 100644
--- a/modules/sfp_emailrep.py
+++ b/modules/sfp_emailrep.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_emailrep(SpiderFootPlugin):
"""EmailRep:Footprint,Investigate,Passive:Search Engines:apikey:Search EmailRep.io for email address reputation."""
@@ -20,9 +20,9 @@ class sfp_emailrep(SpiderFootPlugin):
meta = {
'name': "EmailRep",
'summary': "Search EmailRep.io for email address reputation.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://emailrep.io/",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_emergingthreats.py b/modules/sfp_emergingthreats.py
index bfe0fa2e..b07ec313 100644
--- a/modules/sfp_emergingthreats.py
+++ b/modules/sfp_emergingthreats.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress, IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_emergingthreats(SpiderFootPlugin):
"""Emerging Threats:Investigate,Passive:Reputation Systems::Check if a netblock or IP is malicious according to emergingthreats.net."""
@@ -19,9 +19,9 @@ class sfp_emergingthreats(SpiderFootPlugin):
meta = {
'name': "Emerging Threats",
'summary': "Check if a netblock or IP is malicious according to emergingthreats.net.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://rules.emergingthreats.net/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_errors.py b/modules/sfp_errors.py
index b2048102..792c3c0b 100644
--- a/modules/sfp_errors.py
+++ b/modules/sfp_errors.py
@@ -12,17 +12,17 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
# Taken from Google Dorks on exploit-db.com
regexps = dict({
"PHP Error": ["PHP pase error", "PHP warning", "PHP error",
"unexpected T_VARIABLE", "warning: failed opening", "include_path="],
"Generic Error": ["Internal Server Error", "Incorrect syntax"],
- "Oracle Error": ["ORA-\d+", "TNS:.?no listen"],
+ "Oracle Error": [r"ORA-\d+", "TNS:.?no listen"],
"ASP Error": ["NET_SessionId"],
- "MySQL Error": ["mysql_query\(", "mysql_connect\("],
- "ODBC Error": ["\[ODBC SQL"]
+ "MySQL Error": [r"mysql_query\(", r"mysql_connect\("],
+ "ODBC Error": [r"\[ODBC SQL"]
})
@@ -32,9 +32,9 @@ class sfp_errors(SpiderFootPlugin):
meta = {
'name': "Error String Extractor",
'summary': "Identify common error messages in content like SQL errors, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_ethereum.py b/modules/sfp_ethereum.py
index 4d712cde..99c1b223 100644
--- a/modules/sfp_ethereum.py
+++ b/modules/sfp_ethereum.py
@@ -13,7 +13,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_ethereum(SpiderFootPlugin):
"""Ethereum Address Extractor:Footprint,Investigate,Passive:Content Analysis::Identify ethereum addresses in scraped webpages."""
@@ -21,9 +21,9 @@ class sfp_ethereum(SpiderFootPlugin):
meta = {
'name': "Ethereum Address Extractor",
'summary': "Identify ethereum addresses in scraped webpages.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_filemeta.py b/modules/sfp_filemeta.py
index df4cc24e..19040779 100644
--- a/modules/sfp_filemeta.py
+++ b/modules/sfp_filemeta.py
@@ -17,7 +17,7 @@ import docx
import pptx
import exifread
import io
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_filemeta(SpiderFootPlugin):
@@ -26,9 +26,9 @@ class sfp_filemeta(SpiderFootPlugin):
meta = {
'name': "File Metadata Extractor",
'summary': "Extracts meta data from documents and images.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.py
index a3f676cc..26303d5c 100644
--- a/modules/sfp_fortinet.py
+++ b/modules/sfp_fortinet.py
@@ -12,12 +12,12 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'Fortiguard Threat Lookup': {
'id': '_fortiguard',
- 'checks': [ 'ip' ],
+ 'checks': ['ip'],
'url': 'https://fortiguard.com/search?q={0}&engine=8',
'badregex': ['.*Your signature is blacklisted.*'],
'goodregex': ['.*Your signature is not blacklisted.*']
@@ -31,9 +31,9 @@ class sfp_fortinet(SpiderFootPlugin):
meta = {
'name': "Fortiguard.com",
'summary': "Check if an IP is malicious according to Fortiguard.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://fortiguard.com/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_fraudguard.py b/modules/sfp_fraudguard.py
index 92c02a5a..924b3a92 100644
--- a/modules/sfp_fraudguard.py
+++ b/modules/sfp_fraudguard.py
@@ -15,7 +15,7 @@ import base64
from datetime import datetime
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_fraudguard(SpiderFootPlugin):
"""Fraudguard:Investigate,Passive:Reputation Systems:apikey:Obtain threat information from Fraudguard.io"""
@@ -23,9 +23,9 @@ class sfp_fraudguard(SpiderFootPlugin):
meta = {
'name': "Fraudguard",
'summary': "Obtain threat information from Fraudguard.io",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://fraudguard.io/",
'model': "FREE_AUTH_LIMITED",
@@ -89,7 +89,7 @@ class sfp_fraudguard(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return [ "GEOINFO", "MALICIOUS_IPADDR", "MALICIOUS_NETBLOCK" ]
+ return ["GEOINFO", "MALICIOUS_IPADDR", "MALICIOUS_NETBLOCK"]
def query(self, qry):
fraudguard_url = "https://api.fraudguard.io/ip/" + qry
@@ -104,10 +104,10 @@ class sfp_fraudguard(SpiderFootPlugin):
'Authorization': "Basic " + token.decode('utf-8')
}
- res = self.sf.fetchUrl(fraudguard_url , timeout=self.opts['_fetchtimeout'],
+ res = self.sf.fetchUrl(fraudguard_url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot", headers=headers)
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("Fraudguard.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)
self.errorState = True
return None
@@ -119,7 +119,7 @@ class sfp_fraudguard(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from Fraudguard.io.", False)
+ self.sf.error(f"Error processing JSON response from Fraudguard.io: {e}", False)
return None
#print str(info)
diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.py
index 1ab075c8..6735187d 100644
--- a/modules/sfp_fringeproject.py
+++ b/modules/sfp_fringeproject.py
@@ -15,7 +15,7 @@ import re
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_fringeproject(SpiderFootPlugin):
"""Fringe Project:Investigate,Footprint,Passive:Search Engines::Obtain network information from Fringe Project API."""
@@ -23,9 +23,9 @@ class sfp_fringeproject(SpiderFootPlugin):
meta = {
'name': "Fringe Project",
'summary': "Obtain network information from Fringe Project API.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Footprint", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Footprint", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://fringeproject.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -160,7 +160,7 @@ class sfp_fringeproject(SpiderFootPlugin):
for tag in tags:
try:
port = re.findall(r'^port:([0-9]+)', tag)
- except BaseException as e:
+ except BaseException:
self.sf.debug("Didn't get sane data from FringeProject.")
continue
diff --git a/modules/sfp_fsecure_riddler.py b/modules/sfp_fsecure_riddler.py
index c0d3c520..96e87d46 100644
--- a/modules/sfp_fsecure_riddler.py
+++ b/modules/sfp_fsecure_riddler.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_fsecure_riddler(SpiderFootPlugin):
"""F-Secure Riddler.io:Investigate,Footprint,Passive:Search Engines:apikey:Obtain network information from F-Secure Riddler.io API."""
@@ -20,9 +20,9 @@ class sfp_fsecure_riddler(SpiderFootPlugin):
meta = {
'name': "F-Secure Riddler.io",
'summary': "Obtain network information from F-Secure Riddler.io API.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Footprint", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Footprint", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://riddler.io/",
'model': "PRIVATE_ONLY",
@@ -139,7 +139,7 @@ class sfp_fsecure_riddler(SpiderFootPlugin):
time.sleep(1)
- if res['code'] in [ "400", "401", "402", "403" ]:
+ if res['code'] in ["400", "401", "402", "403"]:
self.sf.error('Unexpected HTTP response code: ' + res['code'], False)
self.errorState = True
return None
diff --git a/modules/sfp_fullcontact.py b/modules/sfp_fullcontact.py
index a3dc12b8..b5cb2def 100644
--- a/modules/sfp_fullcontact.py
+++ b/modules/sfp_fullcontact.py
@@ -12,7 +12,7 @@
import json
import time
from datetime import datetime
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_fullcontact(SpiderFootPlugin):
"""FullContact:Footprint,Investigate,Passive:Search Engines:apikey:Gather domain and e-mail information from fullcontact.com."""
@@ -20,9 +20,9 @@ class sfp_fullcontact(SpiderFootPlugin):
meta = {
'name': "FullContact",
'summary': "Gather domain and e-mail information from fullcontact.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.fullcontact.com",
'model': "FREE_AUTH_LIMITED",
@@ -83,12 +83,12 @@ class sfp_fullcontact(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "DOMAIN_NAME", "EMAILADDR" ]
+ return ["DOMAIN_NAME", "EMAILADDR"]
# What events this module produces
def producedEvents(self):
- return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
- "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS" ]
+ return ["EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
+ "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS"]
def query(self, url, data, failcount=0):
header = "Bearer " + self.opts['api_key']
@@ -98,7 +98,7 @@ class sfp_fullcontact(SpiderFootPlugin):
useragent="SpiderFoot", postData=json.dumps(data),
headers={"Authorization": header})
- if res['code'] in [ "401", "400" ]:
+ if res['code'] in ["401", "400"]:
self.sf.error("API key rejected by fullcontact.com", False)
self.errorState = True
return None
diff --git a/modules/sfp_github.py b/modules/sfp_github.py
index 396235bd..cd870a93 100644
--- a/modules/sfp_github.py
+++ b/modules/sfp_github.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_github(SpiderFootPlugin):
"""Github:Footprint,Passive:Social Media::Identify associated public code repositories on Github."""
@@ -20,9 +20,9 @@ class sfp_github(SpiderFootPlugin):
meta = {
'name': "Github",
'summary': "Identify associated public code repositories on Github.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://github.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -38,12 +38,12 @@ class sfp_github(SpiderFootPlugin):
# Default options
opts = {
- 'namesonly': True
+ 'namesonly': True
}
# Option descriptions
optdescs = {
- 'namesonly': "Match repositories by name only, not by their descriptions. Helps reduce false positives."
+ 'namesonly': "Match repositories by name only, not by their descriptions. Helps reduce false positives."
}
def setup(self, sfc, userOpts=dict()):
@@ -85,7 +85,6 @@ class sfp_github(SpiderFootPlugin):
def handleEvent(self, event):
eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if eventData in self.results:
@@ -111,7 +110,7 @@ class sfp_github(SpiderFootPlugin):
try:
bits = url.split("/")
name = bits[len(bits)-1]
- except BaseException as e:
+ except BaseException:
self.sf.debug("Couldn't get a username out of " + url)
return None
@@ -123,7 +122,7 @@ class sfp_github(SpiderFootPlugin):
try:
json_data = json.loads(res['content'])
except BaseException as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if not json_data.get('login'):
@@ -161,6 +160,7 @@ class sfp_github(SpiderFootPlugin):
self.sf.debug("Looking at " + name)
failed = False
+
# Get all the repositories based on direct matches with the
# name identified
url = "https://api.github.com/search/repositories?q=" + name
@@ -174,11 +174,11 @@ class sfp_github(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except BaseException as e:
+ self.sf.debug(f"Error processing JSON response from GitHub: {e}", False)
ret = None
- if ret == None:
- self.sf.error("Unable to process empty response from Github for: " + \
- name, False)
+ if ret is None:
+ self.sf.error(f"Unable to process empty response from Github for: {name}", False)
failed = True
if not failed:
@@ -211,12 +211,10 @@ class sfp_github(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
if ret == None:
- self.sf.error("Unable to process empty response from Github for: " + \
- name, False)
+ self.sf.error(f"Unable to process empty response from Github for: {name}", False)
failed = True
- except BaseException as e:
- self.sf.error("Unable to process invalid response from Github for: " + \
- name, False)
+ except BaseException:
+ self.sf.error(f"Unable to process invalid response from Github for: {name}", False)
failed = True
if not failed:
@@ -242,7 +240,7 @@ class sfp_github(SpiderFootPlugin):
try:
repret = json.loads(res['content'])
except BaseException as e:
- self.sf.error("Invalid JSON returned from Github.", False)
+ self.sf.error(f"Invalid JSON returned from Github: {e}", False)
continue
if repret == None:
diff --git a/modules/sfp_googlemaps.py b/modules/sfp_googlemaps.py
index d9316b6a..fa68e6a7 100644
--- a/modules/sfp_googlemaps.py
+++ b/modules/sfp_googlemaps.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_googlemaps(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_googlemaps(SpiderFootPlugin):
meta = {
'name': "Google Maps",
'summary': "Identifies potential physical addresses and latitude/longitude coordinates.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Real World" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Real World"],
'dataSource': {
'website': "https://cloud.google.com/maps-platform/",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_googleobjectstorage.py b/modules/sfp_googleobjectstorage.py
index 1e14fcf9..6e00f831 100644
--- a/modules/sfp_googleobjectstorage.py
+++ b/modules/sfp_googleobjectstorage.py
@@ -13,7 +13,7 @@
import threading
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_googleobjectstorage(SpiderFootPlugin):
"""Google Object Storage Finder:Footprint,Passive:Crawling and Scanning::Search for potential Google Object Storage buckets associated with the target and attempt to list their contents."""
@@ -21,9 +21,9 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
meta = {
'name': "Google Object Storage Finder",
'summary': "Search for potential Google Object Storage buckets associated with the target and attempt to list their contents.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Crawling and Scanning" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Crawling and Scanning"],
'dataSource': {
'website': "https://cloud.google.com/storage",
'model': "FREE_NOAUTH_UNLIMITED"
@@ -68,7 +68,7 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
- if res['code'] not in [ "301", "302", "200" ] and \
+ if res['code'] not in ["301", "302", "200"] and \
(res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
@@ -80,7 +80,6 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
self.gosresults[url] = 0
def threadSites(self, siteList):
- ret = list()
self.gosresults = dict()
running = True
i = 0
@@ -154,7 +153,7 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
self.notifyListeners(evt)
return None
- targets = [ eventData.replace('.', '') ]
+ targets = [eventData.replace('.', '')]
kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
if kw:
targets.append(kw)
diff --git a/modules/sfp_googlesafebrowsing.py b/modules/sfp_googlesafebrowsing.py
index 99acca38..df7567e3 100644
--- a/modules/sfp_googlesafebrowsing.py
+++ b/modules/sfp_googlesafebrowsing.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_googlesafebrowsing(SpiderFootPlugin):
@@ -161,7 +161,7 @@ class sfp_googlesafebrowsing(SpiderFootPlugin):
return None
except Exception as e:
- self.sf.error("Error processing JSON response from SHODAN.", False)
+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)
return None
return info
diff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.py
index 88e51d4d..d8d926ae 100644
--- a/modules/sfp_googlesearch.py
+++ b/modules/sfp_googlesearch.py
@@ -10,7 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_googlesearch(SpiderFootPlugin):
@@ -19,9 +19,9 @@ class sfp_googlesearch(SpiderFootPlugin):
meta = {
'name': "Google",
'summary': "Obtain information from the Google Custom Search API to identify sub-domains and links.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://developers.google.com/custom-search",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index 54b5f8ee..0c7b930e 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -14,7 +14,7 @@
import json
import hashlib
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_gravatar(SpiderFootPlugin):
"""Gravatar:Footprint,Investigate,Passive:Social Media::Retrieve user information from Gravatar API."""
@@ -22,9 +22,9 @@ class sfp_gravatar(SpiderFootPlugin):
meta = {
'name': "Gravatar",
'summary': "Retrieve user information from Gravatar API.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://secure.gravatar.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -74,7 +74,7 @@ class sfp_gravatar(SpiderFootPlugin):
# https://secure.gravatar.com/site/implement/
# https://secure.gravatar.com/site/implement/profiles/
def query(self, qry):
- email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest() # nosec
+ email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest() # nosec
output = 'json'
res = self.sf.fetchUrl("https://secure.gravatar.com/" + email_hash + '.' + output,
diff --git a/modules/sfp_greensnow.py b/modules/sfp_greensnow.py
index b75d4025..a9def062 100644
--- a/modules/sfp_greensnow.py
+++ b/modules/sfp_greensnow.py
@@ -12,7 +12,7 @@
from netaddr import IPAddress, IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_greensnow(SpiderFootPlugin):
"""Greensnow:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is malicious according to greensnow.co."""
@@ -20,9 +20,9 @@ class sfp_greensnow(SpiderFootPlugin):
meta = {
'name': "Greensnow",
'summary': "Check if a netblock or IP address is malicious according to greensnow.co.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://greensnow.co/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_grep_app.py b/modules/sfp_grep_app.py
index e7c7c737..660c3ef5 100644
--- a/modules/sfp_grep_app.py
+++ b/modules/sfp_grep_app.py
@@ -15,7 +15,7 @@ import json
import math
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_grep_app(SpiderFootPlugin):
"""grep.app:Footprint,Investigate,Passive:Search Engines::Search grep.app API for links and emails related to the specified domain."""
@@ -23,9 +23,9 @@ class sfp_grep_app(SpiderFootPlugin):
meta = {
'name': "grep.app",
'summary': "Search grep.app API for links and emails related to the specified domain.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://grep.app/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_greynoise.py b/modules/sfp_greynoise.py
index b65ff455..eea93375 100644
--- a/modules/sfp_greynoise.py
+++ b/modules/sfp_greynoise.py
@@ -14,7 +14,7 @@ import json
from datetime import datetime
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_greynoise(SpiderFootPlugin):
"""Greynoise:Investigate,Passive:Reputation Systems:apikey:Obtain information from Greynoise.io's Enterprise API."""
@@ -22,9 +22,9 @@ class sfp_greynoise(SpiderFootPlugin):
meta = {
'name': "Greynoise",
'summary': "Obtain information from Greynoise.io's Enterprise API.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://greynoise.io/",
'model': "FREE_AUTH_LIMITED",
@@ -97,17 +97,15 @@ class sfp_greynoise(SpiderFootPlugin):
return ["MALICIOUS_IPADDR", "MALICIOUS_ASN", "MALICIOUS_SUBNET",
"MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK",
"COMPANY_NAME", "GEOINFO", "BGP_AS_MEMBER", "OPERATING_SYSTEM",
- "RAW_RIR_DATA" ]
+ "RAW_RIR_DATA"]
def queryIP(self, qry):
- ret = None
-
- header = { "key": self.opts['api_key'] }
+ header = {"key": self.opts['api_key']}
url = "https://enterprise.api.greynoise.io/v2/experimental/gnql?query=" + qry
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot", headers=header)
- if res['code'] not in [ "200" ]:
+ if res['code'] not in ["200"]:
self.sf.error("Greynoise API key seems to have been rejected or you have exceeded usage limits.", False)
self.errorState = True
return None
@@ -115,7 +113,7 @@ class sfp_greynoise(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from Greynoise.", False)
+ self.sf.error(f"Error processing JSON response from Greynoise: {e}", False)
return None
return info
diff --git a/modules/sfp_h1nobbdde.py b/modules/sfp_h1nobbdde.py
index 06c9c7d1..3a7ba092 100644
--- a/modules/sfp_h1nobbdde.py
+++ b/modules/sfp_h1nobbdde.py
@@ -11,7 +11,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_h1nobbdde(SpiderFootPlugin):
"""HackerOne (Unofficial):Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed."""
@@ -19,9 +19,9 @@ class sfp_h1nobbdde(SpiderFootPlugin):
meta = {
'name': "HackerOne (Unofficial)",
'summary': "Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "http://www.nobbd.de/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -68,7 +68,6 @@ class sfp_h1nobbdde(SpiderFootPlugin):
# Query h1.nobbd.de
def queryOBB(self, qry):
ret = list()
- base = "http://www.h1.nobbd.de"
url = "http://h1.nobbd.de/search.php?q=" + qry
res = self.sf.fetchUrl(url, timeout=30, useragent=self.opts['_useragent'])
diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.py
index 00ee6571..961d3c21 100644
--- a/modules/sfp_hackertarget.py
+++ b/modules/sfp_hackertarget.py
@@ -16,7 +16,7 @@ import re
import json
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_hackertarget(SpiderFootPlugin):
@@ -25,9 +25,9 @@ class sfp_hackertarget(SpiderFootPlugin):
meta = {
'name': "HackerTarget",
'summary': "Search HackerTarget.com for hosts sharing the same IP.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "https://hackertarget.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -247,7 +247,7 @@ class sfp_hackertarget(SpiderFootPlugin):
# Try and pull out individual records
for row in records:
- pat = re.compile("^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)
+ pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)
grps = re.findall(pat, row)
if len(grps) == 0:
diff --git a/modules/sfp_hashes.py b/modules/sfp_hashes.py
index 7d68e2f8..2817e9da 100644
--- a/modules/sfp_hashes.py
+++ b/modules/sfp_hashes.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_hashes(SpiderFootPlugin):
"""Hash Extractor:Footprint,Investigate,Passive:Content Analysis::Identify MD5 and SHA hashes in web content, files and more."""
@@ -19,9 +19,9 @@ class sfp_hashes(SpiderFootPlugin):
meta = {
'name': "Hash Extractor",
'summary': "Identify MD5 and SHA hashes in web content, files and more.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -60,7 +60,6 @@ class sfp_hashes(SpiderFootPlugin):
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
hashes = self.sf.parseHashes(eventData)
- myres = list()
for hashtup in hashes:
hashalgo, hashval = hashtup
diff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.py
index 79d65bb0..e8bce9fa 100644
--- a/modules/sfp_haveibeenpwned.py
+++ b/modules/sfp_haveibeenpwned.py
@@ -11,7 +11,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_haveibeenpwned(SpiderFootPlugin):
"""HaveIBeenPwned:Footprint,Investigate,Passive:Leaks, Dumps and Breaches:apikey:Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches."""
@@ -19,9 +19,9 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
meta = {
'name': "HaveIBeenPwned",
'summary': "Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://haveibeenpwned.com/",
'model': "COMMERCIAL_ONLY",
@@ -74,12 +74,12 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
def query(self, qry):
ret = None
if self.opts['api_key']:
- v = "3"
+ version = "3"
else:
- v = "2"
+ version = "2"
- url = "https://haveibeenpwned.com/api/v" + v + "/breachedaccount/" + qry
- hdrs = { "Accept": "application/vnd.haveibeenpwned.v" + v + "+json" }
+ url = f"https://haveibeenpwned.com/api/v{version}/breachedaccount/{qry}"
+ hdrs = {"Accept": f"application/vnd.haveibeenpwned.v{version}+json"}
retry = 0
if self.opts['api_key']:
@@ -110,7 +110,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from HaveIBeenPwned?: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from HaveIBeenPwned?: {e}", False)
return None
return ret
@@ -144,7 +144,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
else:
site = n["Name"]
except BaseException as e:
- self.sf.debug("Unable to parse result from HaveIBeenPwned?")
+ self.sf.debug(f"Unable to parse result from HaveIBeenPwned?: {e}")
continue
evt = eventName + "_COMPROMISED"
diff --git a/modules/sfp_honeypot.py b/modules/sfp_honeypot.py
index 93b1d50c..0a400471 100644
--- a/modules/sfp_honeypot.py
+++ b/modules/sfp_honeypot.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_honeypot(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_honeypot(SpiderFootPlugin):
meta = {
'name': "Honeypot Checker",
'summary': "Query the projecthoneypot.org database for entries.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.projecthoneypot.org/",
'model': "FREE_AUTH_UNLIMITED",
@@ -177,7 +177,6 @@ class sfp_honeypot(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
parentEvent = event
- addrlist = list()
if self.errorState:
return None
diff --git a/modules/sfp_hosting.py b/modules/sfp_hosting.py
index 3a3ac170..0840d9e8 100644
--- a/modules/sfp_hosting.py
+++ b/modules/sfp_hosting.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_hosting(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_hosting(SpiderFootPlugin):
meta = {
'name': "Hosting Provider Identifier",
'summary': "Find out if any IP addresses identified fall within known 3rd party hosting ranges, e.g. Amazon, Azure, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -73,7 +73,7 @@ class sfp_hosting(SpiderFootPlugin):
continue
try:
[start, end, title, url] = line.split(",")
- except BaseException as e:
+ except BaseException:
continue
try:
diff --git a/modules/sfp_hunter.py b/modules/sfp_hunter.py
index b307e7e1..a94bea1c 100644
--- a/modules/sfp_hunter.py
+++ b/modules/sfp_hunter.py
@@ -11,7 +11,7 @@
import json
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_hunter(SpiderFootPlugin):
"""Hunter.io:Footprint,Investigate,Passive:Search Engines:apikey:Check for e-mail addresses and names on hunter.io."""
@@ -19,9 +19,9 @@ class sfp_hunter(SpiderFootPlugin):
meta = {
'name': "Hunter.io",
'summary': "Check for e-mail addresses and names on hunter.io.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://hunter.io/",
'model': "FREE_AUTH_LIMITED",
@@ -73,11 +73,11 @@ class sfp_hunter(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "DOMAIN_NAME", "INTERNET_NAME" ]
+ return ["DOMAIN_NAME", "INTERNET_NAME"]
# What events this module produces
def producedEvents(self):
- return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA" ]
+ return ["EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA"]
def query(self, qry, offset=0, limit=10):
params = {
@@ -91,8 +91,7 @@ class sfp_hunter(SpiderFootPlugin):
url = "https://api.hunter.io/v2/domain-search?%s" % urllib.parse.urlencode(params)
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent="SpiderFoot")
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")
if res['code'] == "404":
return None
diff --git a/modules/sfp_hybrid_analysis.py b/modules/sfp_hybrid_analysis.py
index cb9e6d04..4e7f9ca2 100644
--- a/modules/sfp_hybrid_analysis.py
+++ b/modules/sfp_hybrid_analysis.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_hybrid_analysis(SpiderFootPlugin):
"""Hybrid Analysis:Footprint,Investigate,Passive:Reputation Systems:apikey:Search Hybrid Analysis for domains and URLs related to the target."""
@@ -20,9 +20,9 @@ class sfp_hybrid_analysis(SpiderFootPlugin):
meta = {
'name': "Hybrid Analysis",
'summary': "Search Hybrid Analysis for domains and URLs related to the target.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "www.hybrid-analysis.com",
'model': "FREE_AUTH_UNLIMITED",
@@ -144,7 +144,7 @@ class sfp_hybrid_analysis(SpiderFootPlugin):
"hash": qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
headers = {
- "Accept" : "application/json",
+ "Accept": "application/json",
'api-key': self.opts['api_key']
}
res = self.sf.fetchUrl(
@@ -187,7 +187,7 @@ class sfp_hybrid_analysis(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_iban.py b/modules/sfp_iban.py
index a59c1e6b..3cd37297 100644
--- a/modules/sfp_iban.py
+++ b/modules/sfp_iban.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_iban(SpiderFootPlugin):
"""IBAN Number Extractor:Footprint,Investigate,Passive:Content Analysis::Identify IBAN Numbers in any data"""
@@ -19,9 +19,9 @@ class sfp_iban(SpiderFootPlugin):
meta = {
'name': "IBAN Number Extractor",
'summary': "Identify IBAN Numbers in any data",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options.
diff --git a/modules/sfp_iknowwhatyoudownload.py b/modules/sfp_iknowwhatyoudownload.py
index d41066f7..78d84fe9 100644
--- a/modules/sfp_iknowwhatyoudownload.py
+++ b/modules/sfp_iknowwhatyoudownload.py
@@ -10,7 +10,7 @@
#-------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_iknowwhatyoudownload(SpiderFootPlugin):
"""Iknowwhatyoudownload.com:Footprint,Investigate,Passive:Secondary Networks:apikey:Check iknowwhatyoudownload.com for IP addresses that have been using BitTorrent."""
@@ -18,9 +18,9 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):
meta = {
'name': "Iknowwhatyoudownload.com",
'summary': "Check iknowwhatyoudownload.com for IP addresses that have been using BitTorrent.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Secondary Networks" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Secondary Networks"],
'dataSource': {
'website': "https://iknowwhatyoudownload.com/en/peer/",
'model': "FREE_AUTH_LIMITED",
@@ -91,8 +91,7 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):
url = base + qry + "&days=" + str(self.opts['daysback'])
url += "&key=" + self.opts['api_key']
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent="SpiderFoot")
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")
if res['code'] in ["403", "500"]:
self.sf.info("Unable to fetch data from iknowwhatyoudownload.com right now.")
diff --git a/modules/sfp_instagram.py b/modules/sfp_instagram.py
index d159208a..b52a4c7c 100644
--- a/modules/sfp_instagram.py
+++ b/modules/sfp_instagram.py
@@ -12,7 +12,7 @@
import json
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_instagram(SpiderFootPlugin):
"""Instagram:Footprint,Investigate,Passive:Social Media::Gather information from Instagram profiles."""
@@ -20,9 +20,9 @@ class sfp_instagram(SpiderFootPlugin):
meta = {
'name': "Instagram",
'summary': "Gather information from Instagram profiles.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://www.instagram.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -55,11 +55,11 @@ class sfp_instagram(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ 'SOCIAL_MEDIA' ]
+ return ['SOCIAL_MEDIA']
# What events this module produces
def producedEvents(self):
- return [ 'RAW_RIR_DATA' ]
+ return ['RAW_RIR_DATA']
# Extract profile JSON from HTML
def extractJson(self, html):
diff --git a/modules/sfp_intelx.py b/modules/sfp_intelx.py
index bb3264d8..afc0764a 100644
--- a/modules/sfp_intelx.py
+++ b/modules/sfp_intelx.py
@@ -14,7 +14,7 @@
import json
import time
import datetime
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_intelx(SpiderFootPlugin):
@@ -23,9 +23,9 @@ class sfp_intelx(SpiderFootPlugin):
meta = {
'name': "IntelligenceX",
'summary': "Obtain information from IntelligenceX about identified IP addresses, domains, e-mail addresses and phone numbers.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://intelx.io/",
'model': "FREE_AUTH_LIMITED",
@@ -252,15 +252,15 @@ class sfp_intelx(SpiderFootPlugin):
try:
val = rec['selectorvalueh']
evt = None
- if rec['selectortype'] == 1: #Email
+ if rec['selectortype'] == 1: # Email
evt = "EMAILADDR"
if val.split("@")[0] in self.opts['_genericusers'].split(","):
evt = "EMAILADDR_GENERIC"
- if rec['selectortype'] == 2: #Domain
+ if rec['selectortype'] == 2: # Domain
evt = "INTERNET_NAME"
if val == eventData:
continue
- if rec['selectortype'] == 3: #URL
+ if rec['selectortype'] == 3: # URL
evt = "LINKED_URL_INTERNAL"
if not val or not evt:
diff --git a/modules/sfp_intfiles.py b/modules/sfp_intfiles.py
index b7c630dd..88781812 100644
--- a/modules/sfp_intfiles.py
+++ b/modules/sfp_intfiles.py
@@ -10,7 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_intfiles(SpiderFootPlugin):
@@ -19,9 +19,9 @@ class sfp_intfiles(SpiderFootPlugin):
meta = {
'name': "Interesting File Finder",
'summary': "Identifies potential files of interest, e.g. office documents, zip files.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Crawling and Scanning" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Crawling and Scanning"]
}
# Default options
diff --git a/modules/sfp_ipinfo.py b/modules/sfp_ipinfo.py
index 8213b776..c5230a5f 100644
--- a/modules/sfp_ipinfo.py
+++ b/modules/sfp_ipinfo.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_ipinfo(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_ipinfo(SpiderFootPlugin):
meta = {
'name': "IPInfo.io",
'summary': "Identifies the physical location of IP addresses identified using ipinfo.io.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Real World" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Real World"],
'dataSource': {
'website': "https://ipinfo.io",
'model': "FREE_AUTH_LIMITED",
@@ -95,7 +95,7 @@ class sfp_ipinfo(SpiderFootPlugin):
try:
result = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return result
diff --git a/modules/sfp_ipstack.py b/modules/sfp_ipstack.py
index 979507bc..e1079dc2 100644
--- a/modules/sfp_ipstack.py
+++ b/modules/sfp_ipstack.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_ipstack(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_ipstack(SpiderFootPlugin):
meta = {
'name': "ipstack",
'summary': "Identifies the physical location of IP addresses identified using ipstack.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Real World" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Real World"],
'dataSource': {
'website': "https://ipstack.com/",
'model': "FREE_AUTH_LIMITED",
@@ -108,7 +108,7 @@ class sfp_ipstack(SpiderFootPlugin):
self.errorState = True
return None
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if hostip.get('country_name', None) != None:
diff --git a/modules/sfp_isc.py b/modules/sfp_isc.py
index 71b5a386..42c96d8b 100644
--- a/modules/sfp_isc.py
+++ b/modules/sfp_isc.py
@@ -12,14 +12,14 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'Internet Storm Center': {
'id': '_isc',
'checks': ['ip'],
'url': 'https://isc.sans.edu/api/ip/{0}',
- 'badregex': ['.*<attacks>\d+</attacks>.*'],
+ 'badregex': [r'.*<attacks>\d+</attacks>.*'],
'goodregex': []
}
}
@@ -31,9 +31,9 @@ class sfp_isc(SpiderFootPlugin):
meta = {
'name': "Internet Storm Center",
'summary': "Check if an IP is malicious according to SANS ISC.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://isc.sans.edu",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -87,7 +87,7 @@ class sfp_isc(SpiderFootPlugin):
# What events is this module interested in for input
# * = be notified about all events.
def watchedEvents(self):
- return ["IP_ADDRESS", "AFFILIATE_IPADDR" ]
+ return ["IP_ADDRESS", "AFFILIATE_IPADDR"]
# What events this module produces
# This is to support the end user in selecting modules based on events
diff --git a/modules/sfp_jsonwhoiscom.py b/modules/sfp_jsonwhoiscom.py
index 8a365acd..18cac5f5 100644
--- a/modules/sfp_jsonwhoiscom.py
+++ b/modules/sfp_jsonwhoiscom.py
@@ -13,7 +13,7 @@
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_jsonwhoiscom(SpiderFootPlugin):
"""JsonWHOIS.com:Footprint,Investigate,Passive:Search Engines:apikey:Search JsonWHOIS.com for WHOIS records associated with a domain."""
@@ -21,9 +21,9 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
meta = {
'name': "JsonWHOIS.com",
'summary': "Search JsonWHOIS.com for WHOIS records associated with a domain.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://jsonwhois.com",
'model': "FREE_AUTH_LIMITED",
@@ -86,8 +86,8 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
headers = {
- "Accept" : "application/json",
- "Authorization" : "Token token=" + self.opts["api_key"]
+ "Accept": "application/json",
+ "Authorization": "Token token=" + self.opts["api_key"]
}
res = self.sf.fetchUrl(
@@ -139,7 +139,7 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.py
index 49cc5af0..f1798608 100644
--- a/modules/sfp_junkfiles.py
+++ b/modules/sfp_junkfiles.py
@@ -10,7 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import random
class sfp_junkfiles(SpiderFootPlugin):
@@ -19,9 +19,9 @@ class sfp_junkfiles(SpiderFootPlugin):
meta = {
'name': "Junk File Finder",
'summary': "Looks for old/temporary and other similar files.",
- 'flags': [ "slow", "errorprone", "invasive" ],
- 'useCases': [ "Footprint" ],
- 'categories': [ "Crawling and Scanning" ]
+ 'flags': ["slow", "errorprone", "invasive"],
+ 'useCases': ["Footprint"],
+ 'categories': ["Crawling and Scanning"]
}
# Default options
diff --git a/modules/sfp_keybase.py b/modules/sfp_keybase.py
index 7a12b8b7..2677be95 100644
--- a/modules/sfp_keybase.py
+++ b/modules/sfp_keybase.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import urllib.request, urllib.parse, urllib.error
import json
import re
@@ -22,9 +22,9 @@ class sfp_keybase(SpiderFootPlugin):
meta = {
'name': "Keybase",
'summary': "Obtain additional information about target username",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Public Registries" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Public Registries"],
'dataSource': {
'website': "https://keybase.io/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -62,9 +62,11 @@ class sfp_keybase(SpiderFootPlugin):
return ["USERNAME", "LINKED_URL_EXTERNAL"]
def producedEvents(self):
- return ["RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME",
+ return [
+ "RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME",
"HUMAN_NAME", "GEOINFO", "BITCOIN_ADDRESS",
- "PGP_KEY"]
+ "PGP_KEY"
+ ]
def queryUsername(self, qry):
@@ -73,7 +75,7 @@ class sfp_keybase(SpiderFootPlugin):
}
headers = {
- 'Accept' : "application/json"
+ 'Accept': "application/json"
}
res = self.sf.fetchUrl(
@@ -129,7 +131,7 @@ class sfp_keybase(SpiderFootPlugin):
# Extract username if a Keybase link is received
if eventName == "LINKED_URL_EXTERNAL":
- linkRegex = "^https?://keybase.io\/[A-Za-z0-9\-_\.]+"
+ linkRegex = r"^https?://keybase.io\/[A-Za-z0-9\-_\.]+"
links = re.findall(linkRegex, eventData)
if len(links) == 0:
@@ -229,9 +231,8 @@ class sfp_keybase(SpiderFootPlugin):
evt = SpiderFootEvent("BITCOIN_ADDRESS", btcAddress, self.__name__, event)
self.notifyListeners(evt)
-
# Extract PGP Keys
- pgpRegex = "-----BEGIN\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----(.*?)-----END\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----"
+ pgpRegex = r"-----BEGIN\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----(.*?)-----END\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----"
pgpKeys = re.findall(pgpRegex, str(content))
diff --git a/modules/sfp_leakix.py b/modules/sfp_leakix.py
index d6627a86..11c17b58 100644
--- a/modules/sfp_leakix.py
+++ b/modules/sfp_leakix.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_leakix(SpiderFootPlugin):
"""LeakIX:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Search LeakIX for host data leaks, open ports, software and geoip."""
@@ -20,9 +20,9 @@ class sfp_leakix(SpiderFootPlugin):
meta = {
'name': "LeakIX",
'summary': "Search LeakIX for host data leaks, open ports, software and geoip.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://leakix.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -73,7 +73,7 @@ class sfp_leakix(SpiderFootPlugin):
# https://leakix.net/api-documentation
def queryHost(self, qry):
headers = {
- "Accept" : "application/json"
+ "Accept": "application/json"
}
res = self.sf.fetchUrl(
'https://leakix.net/host/' + qry,
@@ -110,7 +110,7 @@ class sfp_leakix(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_malwaredomainlist.py b/modules/sfp_malwaredomainlist.py
index 6213ec15..6b0eadeb 100644
--- a/modules/sfp_malwaredomainlist.py
+++ b/modules/sfp_malwaredomainlist.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'malwaredomainlist.com IP List': {
@@ -25,7 +25,7 @@ malchecks = {
'id': 'malwaredomainlistdomain',
'checks': ['domain'],
'url': 'http://www.malwaredomainlist.com/hostslist/hosts.txt',
- 'regex': '.*\s+{0}[\s$]'
+ 'regex': r'.*\s+{0}[\s$]'
}
}
@@ -35,9 +35,9 @@ class sfp_malwaredomainlist(SpiderFootPlugin):
meta = {
'name': "malwaredomainlist.com",
'summary': "Check if a host/domain, IP or netblock is malicious according to malwaredomainlist.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.malwaredomainlist.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -135,8 +135,7 @@ class sfp_malwaredomainlist(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_malwaredomains.py b/modules/sfp_malwaredomains.py
index 88816c8a..d3f4dd5c 100644
--- a/modules/sfp_malwaredomains.py
+++ b/modules/sfp_malwaredomains.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'malwaredomains.com Domains': {
@@ -30,9 +30,9 @@ class sfp_malwaredomains(SpiderFootPlugin):
meta = {
'name': "malwaredomains.com",
'summary': "Check if a host/domain is malicious according to malwaredomains.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.malwaredomains.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -122,8 +122,7 @@ class sfp_malwaredomains(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_malwarepatrol.py b/modules/sfp_malwarepatrol.py
index 9422def4..27d85077 100644
--- a/modules/sfp_malwarepatrol.py
+++ b/modules/sfp_malwarepatrol.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_malwarepatrol(SpiderFootPlugin):
@@ -20,9 +20,9 @@ class sfp_malwarepatrol(SpiderFootPlugin):
meta = {
'name': "MalwarePatrol",
'summary': "Searches malwarepatrol.net's database of malicious URLs/IPs.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.malwarepatrol.net/",
'model': "FREE_AUTH_LIMITED",
@@ -132,7 +132,7 @@ class sfp_malwarepatrol(SpiderFootPlugin):
evtType = 'MALICIOUS_AFFILIATE_IPADDR'
if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',
- 'AFFILIATE_INTERNET_NAME' ]:
+ 'AFFILIATE_INTERNET_NAME']:
if eventName == "INTERNET_NAME":
evtType = "MALICIOUS_INTERNET_NAME"
if eventName == 'AFFILIATE_INTERNET_NAME':
diff --git a/modules/sfp_metadefender.py b/modules/sfp_metadefender.py
index bbd66a7d..6524e056 100644
--- a/modules/sfp_metadefender.py
+++ b/modules/sfp_metadefender.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_metadefender(SpiderFootPlugin):
"""MetaDefender:Footprint,Investigate,Passive:Reputation Systems:apikey:Search MetaDefender API for IP address and domain IP reputation."""
@@ -20,9 +20,9 @@ class sfp_metadefender(SpiderFootPlugin):
meta = {
'name': "MetaDefender",
'summary': "Search MetaDefender API for IP address and domain IP reputation.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ]
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Reputation Systems"]
}
# Default options
@@ -114,7 +114,7 @@ class sfp_metadefender(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return data
diff --git a/modules/sfp_mnemonic.py b/modules/sfp_mnemonic.py
index f99bf48b..dee2e7b8 100644
--- a/modules/sfp_mnemonic.py
+++ b/modules/sfp_mnemonic.py
@@ -14,7 +14,7 @@
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_mnemonic(SpiderFootPlugin):
"""Mnemonic PassiveDNS:Footprint,Investigate,Passive:Passive DNS::Obtain Passive DNS information from PassiveDNS.mnemonic.no."""
@@ -22,9 +22,9 @@ class sfp_mnemonic(SpiderFootPlugin):
meta = {
'name': "Mnemonic PassiveDNS",
'summary': "Obtain Passive DNS information from PassiveDNS.mnemonic.no.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "https://www.mnemonic.no",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -109,7 +109,7 @@ class sfp_mnemonic(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
# Check the response is ok
@@ -167,7 +167,7 @@ class sfp_mnemonic(SpiderFootPlugin):
max_pages = int(self.opts['max_pages'])
per_page = int(self.opts['per_page'])
agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])
- cohostcount = 0
+ self.cohostcount = 0
cohosts = list()
while position < (per_page * max_pages):
@@ -193,11 +193,11 @@ class sfp_mnemonic(SpiderFootPlugin):
self.sf.debug("Record found too old, skipping.")
continue
- if eventName in [ 'IP_ADDRESS' ]:
+ if eventName in ['IP_ADDRESS']:
if r['rrtype'] == 'a':
cohosts.append(r['query'])
- if eventName in [ 'INTERNET_NAME', 'DOMAIN_NAME' ]:
+ if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:
# Ignore PTR records
if r['rrtype'] == 'ptr':
diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.py
index 22904225..d70614c9 100644
--- a/modules/sfp_multiproxy.py
+++ b/modules/sfp_multiproxy.py
@@ -13,7 +13,7 @@
import re
from netaddr import IPAddress, IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'multiproxy.org Open Proxies': {
@@ -31,9 +31,9 @@ class sfp_multiproxy(SpiderFootPlugin):
meta = {
'name': "multiproxy.org Open Proxies",
'summary': "Check if an IP is an open proxy according to multiproxy.org' open proxy list.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Secondary Networks" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Secondary Networks"],
'dataSource': {
'website': "https://multiproxy.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -123,8 +123,7 @@ class sfp_multiproxy(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.py
index 07a0a520..669a887b 100644
--- a/modules/sfp_myspace.py
+++ b/modules/sfp_myspace.py
@@ -11,7 +11,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_myspace(SpiderFootPlugin):
"""MySpace:Footprint,Investigate,Passive:Social Media::Gather username and location from MySpace.com profiles."""
@@ -19,9 +19,9 @@ class sfp_myspace(SpiderFootPlugin):
meta = {
'name': "MySpace",
'summary': "Gather username and location from MySpace.com profiles.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://myspace.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -58,11 +58,11 @@ class sfp_myspace(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "EMAILADDR", "SOCIAL_MEDIA" ]
+ return ["EMAILADDR", "SOCIAL_MEDIA"]
# What events this module produces
def producedEvents(self):
- return [ "SOCIAL_MEDIA", "GEOINFO" ]
+ return ["SOCIAL_MEDIA", "GEOINFO"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -100,8 +100,8 @@ class sfp_myspace(SpiderFootPlugin):
# Check for email address as name, at the risk of missed results.
try:
- matches = re.findall('<a href=\"\/([a-zA-Z0-9_]+)\".*[\&; :\"\#\*\(\"\'\;\,\>\.\?\!]+' + email + '[\&; :\"\#\*\)\"\'\;\,\<\.\?\!]+', profile, re.IGNORECASE)
- except BaseException as e:
+ matches = re.findall(r'<a href=\"\/([a-zA-Z0-9_]+)\".*[\&; :\"\#\*\(\"\'\;\,\>\.\?\!]+' + email + r'[\&; :\"\#\*\)\"\'\;\,\<\.\?\!]+', profile, re.IGNORECASE)
+ except BaseException:
self.sf.debug("Malformed e-mail address, skipping.")
return None
diff --git a/modules/sfp_names.py b/modules/sfp_names.py
index 99fc852f..3d97cba0 100644
--- a/modules/sfp_names.py
+++ b/modules/sfp_names.py
@@ -12,7 +12,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_names(SpiderFootPlugin):
"""Human Name Extractor:Footprint,Passive:Content Analysis:errorprone:Attempt to identify human names in fetched content."""
@@ -20,9 +20,9 @@ class sfp_names(SpiderFootPlugin):
meta = {
'name': "Human Name Extractor",
'summary': "Attempt to identify human names in fetched content.",
- 'flags': [ "errorprone" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': ["errorprone"],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -106,7 +106,7 @@ class sfp_names(SpiderFootPlugin):
# For RAW_RIR_DATA, there are only specific modules we
# expect to see RELEVANT names within.
if eventName == "RAW_RIR_DATA":
- if srcModuleName not in [ "sfp_arin", "sfp_builtwith", "sfp_clearbit",
+ if srcModuleName not in ["sfp_arin", "sfp_builtwith", "sfp_clearbit",
"sfp_fullcontact", "sfp_github", "sfp_hunter",
"sfp_opencorporates", "sfp_slideshare",
"sfp_twitter", "sfp_venmo", "sfp_instagram"]:
@@ -134,7 +134,7 @@ class sfp_names(SpiderFootPlugin):
# If both words are not in the dictionary, add 75 points.
if first not in self.d and second not in self.d:
- self.sf.debug("Both first and second names are not in the dictionary, so high chance of name: (" + first +":" + second +").")
+ self.sf.debug(f"Both first and second names are not in the dictionary, so high chance of name: ({first}:{second}).")
p += 75
notindict = True
else:
diff --git a/modules/sfp_networksdb.py b/modules/sfp_networksdb.py
index b0cec144..88f8ce81 100644
--- a/modules/sfp_networksdb.py
+++ b/modules/sfp_networksdb.py
@@ -14,7 +14,7 @@
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_networksdb(SpiderFootPlugin):
"""NetworksDB:Footprint,Investigate,Passive:Passive DNS:apikey:Search NetworksDB.io API for IP address and domain information."""
@@ -22,9 +22,9 @@ class sfp_networksdb(SpiderFootPlugin):
meta = {
'name': "NetworksDB",
'summary': "Search NetworksDB.io API for IP address and domain information.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "https://networksdb.io/",
'model': "FREE_AUTH_LIMITED",
@@ -233,7 +233,7 @@ class sfp_networksdb(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from NetworksDB.", False)
+ self.sf.error(f"Error processing JSON response from NetworksDB: {e}", False)
return None
if data.get('warning'):
diff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.py
index 5f783f02..80c160f1 100644
--- a/modules/sfp_neutrinoapi.py
+++ b/modules/sfp_neutrinoapi.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_neutrinoapi(SpiderFootPlugin):
"""NeutrinoAPI:Footprint,Investigate,Passive:Reputation Systems:apikey:Search NeutrinoAPI for IP address info and check IP reputation."""
@@ -20,9 +20,9 @@ class sfp_neutrinoapi(SpiderFootPlugin):
meta = {
'name': "NeutrinoAPI",
'summary': "Search NeutrinoAPI for IP address info and check IP reputation.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.neutrinoapi.com/",
'model': "FREE_AUTH_LIMITED",
@@ -82,36 +82,48 @@ class sfp_neutrinoapi(SpiderFootPlugin):
# Query the phone-validate REST API
# https://www.neutrinoapi.com/api/phone-validate/
def queryPhoneValidate(self, qry):
- res = self.sf.fetchUrl('https://neutrinoapi.com/phone-validate',
- postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
+ res = self.sf.fetchUrl(
+ 'https://neutrinoapi.com/phone-validate',
+ postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
+ timeout=self.opts['timeout'],
+ useragent=self.opts['_useragent']
+ )
return self.parseApiResponse(res)
# Query the ip-info REST API
# https://www.neutrinoapi.com/api/ip-info/
def queryIpInfo(self, qry):
- res = self.sf.fetchUrl("https://neutrinoapi.com/ip-info",
- postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
+ res = self.sf.fetchUrl(
+ "https://neutrinoapi.com/ip-info",
+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
+ timeout=self.opts['timeout'],
+ useragent=self.opts['_useragent']
+ )
return self.parseApiResponse(res)
# Query the ip-blocklist REST API
# https://www.neutrinoapi.com/api/ip-blocklist/
def queryIpBlocklist(self, qry):
- res = self.sf.fetchUrl("https://neutrinoapi.com/ip-blocklist",
- postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
+ res = self.sf.fetchUrl(
+ "https://neutrinoapi.com/ip-blocklist",
+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
+ timeout=self.opts['timeout'],
+ useragent=self.opts['_useragent']
+ )
return self.parseApiResponse(res)
# Query the host-reputation REST API
# https://www.neutrinoapi.com/api/host-reputation/
def queryHostReputation(self, qry):
- res = self.sf.fetchUrl("https://neutrinoapi.com/host-reputation",
- postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
+ res = self.sf.fetchUrl(
+ "https://neutrinoapi.com/host-reputation",
+ postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
+ timeout=self.opts['timeout'],
+ useragent=self.opts['_useragent']
+ )
return self.parseApiResponse(res)
@@ -128,7 +140,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if res['code'] == "400":
diff --git a/modules/sfp_norton.py b/modules/sfp_norton.py
index 0ce0aff1..a00c9eda 100644
--- a/modules/sfp_norton.py
+++ b/modules/sfp_norton.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_norton(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_norton(SpiderFootPlugin):
meta = {
'name': "Norton ConnectSafe",
'summary': "Check if a host would be blocked by Norton ConnectSafe DNS",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ]
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"]
}
# Default options
@@ -56,13 +56,13 @@ class sfp_norton(SpiderFootPlugin):
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "199.85.126.20", "199.85.127.20" ]
+ res.nameservers = ["199.85.126.20", "199.85.127.20"]
try:
addrs = res.query(qaddr)
self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -88,7 +88,7 @@ class sfp_norton(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
+ except BaseException:
return None
if resolved:
diff --git a/modules/sfp_numverify.py b/modules/sfp_numverify.py
index 63f8d744..84057a2c 100644
--- a/modules/sfp_numverify.py
+++ b/modules/sfp_numverify.py
@@ -13,7 +13,7 @@
import json
import urllib.request, urllib.parse, urllib.error
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_numverify(SpiderFootPlugin):
"""numverify:Footprint,Investigate,Passive:Real World:apikey:Lookup phone number location and carrier information from numverify.com."""
@@ -21,9 +21,9 @@ class sfp_numverify(SpiderFootPlugin):
meta = {
'name': "numverify",
'summary': "Lookup phone number location and carrier information from numverify.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Real World" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Real World"],
'dataSource': {
'website': "http://numverify.com/",
'model': "FREE_AUTH_LIMITED",
@@ -85,7 +85,7 @@ class sfp_numverify(SpiderFootPlugin):
params = {
'number': number.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'country_code': '',
- 'format': '0', # set to "1" for prettified debug output
+ 'format': '0', # set to "1" for prettified debug output
'access_key': self.opts['api_key']
}
diff --git a/modules/sfp_onioncity.py b/modules/sfp_onioncity.py
index ee7688fd..87be8e50 100644
--- a/modules/sfp_onioncity.py
+++ b/modules/sfp_onioncity.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import re
@@ -21,9 +21,9 @@ class sfp_onioncity(SpiderFootPlugin):
meta = {
'name': "Onion.link",
'summary': "Search Tor 'Onion City' search engine for mentions of the target domain.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Search Engines" ]
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Search Engines"]
}
# Default options
@@ -117,7 +117,7 @@ class sfp_onioncity(SpiderFootPlugin):
timeout=self.opts["_fetchtimeout"],
useragent=self.opts["_useragent"],
)
- if response['code'] in [ "200", "201", "202" ]:
+ if response['code'] in ["200", "201", "202"]:
evt = SpiderFootEvent(
"RAW_RIR_DATA", response["content"], self.__name__, event
)
@@ -146,8 +146,8 @@ class sfp_onioncity(SpiderFootPlugin):
continue
# Sometimes onion city search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
continue
@@ -157,7 +157,7 @@ class sfp_onioncity(SpiderFootPlugin):
try:
startIndex = res['content'].index(eventData) - 120
endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
+ except BaseException:
self.sf.debug("String not found in content.")
continue
diff --git a/modules/sfp_onionsearchengine.py b/modules/sfp_onionsearchengine.py
index 64066efb..7197841a 100644
--- a/modules/sfp_onionsearchengine.py
+++ b/modules/sfp_onionsearchengine.py
@@ -14,7 +14,7 @@
import re
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_onionsearchengine(SpiderFootPlugin):
"""Onionsearchengine.com:Footprint,Investigate:Search Engines::Search Tor onionsearchengine.com for mentions of the target domain."""
@@ -22,9 +22,9 @@ class sfp_onionsearchengine(SpiderFootPlugin):
meta = {
'name': "Onionsearchengine.com",
'summary': "Search Tor onionsearchengine.com for mentions of the target domain.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://as.onionsearchengine.com",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -44,7 +44,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):
'timeout': 10,
'max_pages': 20,
'fetchlinks': True,
- 'blacklist': [ '.*://relate.*' ],
+ 'blacklist': ['.*://relate.*'],
'fullnames': True
}
@@ -78,7 +78,6 @@ class sfp_onionsearchengine(SpiderFootPlugin):
def handleEvent(self, event):
eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':
@@ -128,7 +127,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):
self.__name__, event)
self.notifyListeners(evt)
- links = re.findall("url\.php\?u=(.[^\"\']+)[\"\']",
+ links = re.findall(r"url\.php\?u=(.[^\"\']+)[\"\']",
data['content'], re.IGNORECASE | re.DOTALL)
for link in links:
@@ -177,7 +176,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):
try:
startIndex = res['content'].index(eventData) - 120
endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
+ except BaseException:
self.sf.debug('String "' + eventData + '" not found in content.')
continue
diff --git a/modules/sfp_onyphe.py b/modules/sfp_onyphe.py
index 11af0371..3ca51d4b 100644
--- a/modules/sfp_onyphe.py
+++ b/modules/sfp_onyphe.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import json
from datetime import datetime
import time
diff --git a/modules/sfp_open_passive_dns_database.py b/modules/sfp_open_passive_dns_database.py
index 3a9dc756..cb95621b 100644
--- a/modules/sfp_open_passive_dns_database.py
+++ b/modules/sfp_open_passive_dns_database.py
@@ -14,7 +14,7 @@
import re
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_open_passive_dns_database(SpiderFootPlugin):
"""Open Passive DNS Database:Footprint,Investigate,Passive:Passive DNS::Obtain passive DNS information from pdns.daloo.de Open passive DNS database."""
@@ -22,9 +22,9 @@ class sfp_open_passive_dns_database(SpiderFootPlugin):
meta = {
'name': "Open Passive DNS Database",
'summary': "Obtain passive DNS information from pdns.daloo.de Open passive DNS database.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "http://pdns.daloo.de/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -73,7 +73,7 @@ class sfp_open_passive_dns_database(SpiderFootPlugin):
# Query the Open Passive DNS Database for a domain
def query(self, qry):
params = {
- "alike": 1, # alike is required to find subdomains
+ "alike": 1, # alike is required to find subdomains
"q": qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
diff --git a/modules/sfp_openbugbounty.py b/modules/sfp_openbugbounty.py
index 4a965d00..dc53dbdb 100644
--- a/modules/sfp_openbugbounty.py
+++ b/modules/sfp_openbugbounty.py
@@ -11,7 +11,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_openbugbounty(SpiderFootPlugin):
"""Open Bug Bounty:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed."""
@@ -19,9 +19,9 @@ class sfp_openbugbounty(SpiderFootPlugin):
meta = {
'name': "Open Bug Bounty",
'summary': "Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://www.openbugbounty.org/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.py
index 700bfe09..cc6a988d 100644
--- a/modules/sfp_opencorporates.py
+++ b/modules/sfp_opencorporates.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_opencorporates(SpiderFootPlugin):
"""OpenCorporates:Passive,Footprint,Investigate:Search Engines::Look up company information from OpenCorporates."""
@@ -20,9 +20,9 @@ class sfp_opencorporates(SpiderFootPlugin):
meta = {
'name': "OpenCorporates",
'summary': "Look up company information from OpenCorporates.",
- 'flags': [ "" ],
- 'useCases': [ "Passive", "Footprint", "Investigate" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Passive", "Footprint", "Investigate"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://opencorporates.com",
'model': "FREE_NOAUTH_LIMITED",
@@ -62,11 +62,11 @@ class sfp_opencorporates(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "COMPANY_NAME" ]
+ return ["COMPANY_NAME"]
# What events this module produces
def producedEvents(self):
- return [ "COMPANY_NAME", "PHYSICAL_ADDRESS", "RAW_RIR_DATA" ]
+ return ["COMPANY_NAME", "PHYSICAL_ADDRESS", "RAW_RIR_DATA"]
# Search for company name
# https://api.opencorporates.com/documentation/API-Reference
diff --git a/modules/sfp_opendns.py b/modules/sfp_opendns.py
index 2e952ced..8f297a40 100644
--- a/modules/sfp_opendns.py
+++ b/modules/sfp_opendns.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_opendns(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_opendns(SpiderFootPlugin):
meta = {
'name': "OpenDNS",
'summary': "Check if a host would be blocked by OpenDNS DNS",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.opendns.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -71,13 +71,13 @@ class sfp_opendns(SpiderFootPlugin):
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "208.67.222.222", "208.67.220.220" ]
+ res.nameservers = ["208.67.222.222", "208.67.220.220"]
try:
addrs = res.query(qaddr)
self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -103,7 +103,7 @@ class sfp_opendns(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
+ except BaseException:
return None
if resolved:
diff --git a/modules/sfp_openphish.py b/modules/sfp_openphish.py
index 3de71e3c..e3d36a60 100644
--- a/modules/sfp_openphish.py
+++ b/modules/sfp_openphish.py
@@ -13,14 +13,14 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'OpenPhish': {
'id': '_openphish',
'checks': ['domain'],
'url': 'https://www.openphish.com/feed.txt',
- 'regex': '\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*'
+ 'regex': r'\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*'
}
}
@@ -31,9 +31,9 @@ class sfp_openphish(SpiderFootPlugin):
meta = {
'name': "OpenPhish",
'summary': "Check if a host/domain is malicious according to OpenPhish.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://openphish.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -121,8 +121,7 @@ class sfp_openphish(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
@@ -140,11 +139,10 @@ class sfp_openphish(SpiderFootPlugin):
try:
if IPAddress(ip) in IPNetwork(target):
- self.sf.debug(ip + " found within netblock/subnet " +
- target + " in " + check)
+ self.sf.debug(f"{ip} found within netblock/subnet {target} in {check}")
return url
except Exception as e:
- self.sf.debug("Error encountered parsing: " + str(e))
+ self.sf.debug(f"Error encountered parsing: {e}")
continue
return None
diff --git a/modules/sfp_openstreetmap.py b/modules/sfp_openstreetmap.py
index 05586a9e..25d7af3d 100644
--- a/modules/sfp_openstreetmap.py
+++ b/modules/sfp_openstreetmap.py
@@ -15,7 +15,7 @@ import re
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_openstreetmap(SpiderFootPlugin):
"""OpenStreetMap:Footprint,Investigate,Passive:Real World::Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API."""
@@ -23,9 +23,9 @@ class sfp_openstreetmap(SpiderFootPlugin):
meta = {
'name': "OpenStreetMap",
'summary': "Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Real World" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Real World"],
'dataSource': {
'website': "https://www.openstreetmap.org/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.py
index a860334b..9bd0f53c 100644
--- a/modules/sfp_pageinfo.py
+++ b/modules/sfp_pageinfo.py
@@ -14,7 +14,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
# Indentify pages that use Javascript libs, handle passwords, have forms,
# permit file uploads and more to come.
@@ -34,9 +34,9 @@ class sfp_pageinfo(SpiderFootPlugin):
meta = {
'name': "Page Information",
'summary': "Obtain information about web pages (do they take passwords, do they contain forms, etc.)",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.py
index deb600c9..541f16d5 100644
--- a/modules/sfp_pastebin.py
+++ b/modules/sfp_pastebin.py
@@ -13,7 +13,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_pastebin(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_pastebin(SpiderFootPlugin):
meta = {
'name': "PasteBin",
'summary': "PasteBin scraping (via Google) to identify related content.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://pastebin.com/",
'model': "FREE_AUTH_LIMITED",
@@ -86,8 +86,6 @@ class sfp_pastebin(SpiderFootPlugin):
return ["LEAKSITE_CONTENT", "LEAKSITE_URL"]
def handleEvent(self, event):
- eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if self.errorState:
@@ -104,7 +102,6 @@ class sfp_pastebin(SpiderFootPlugin):
self.results[eventData] = True
for dom in list(self.domains.keys()):
- links = list()
target = self.domains[dom]
res = self.sf.googleIterate(
searchString="+site:{target_site} \"{search_keyword}\"".format(
@@ -148,13 +145,13 @@ class sfp_pastebin(SpiderFootPlugin):
continue
# Sometimes pastes search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
continue
try:
startIndex = res['content'].index(eventData)
- except BaseException as e:
+ except BaseException:
self.sf.debug("String not found in pastes content.")
continue
diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.py
index 1e3c1a2a..c51e8d5a 100644
--- a/modules/sfp_pgp.py
+++ b/modules/sfp_pgp.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_pgp(SpiderFootPlugin):
"""PGP Key Servers:Footprint,Investigate,Passive:Public Registries::Look up e-mail addresses in PGP public key servers."""
@@ -21,9 +21,9 @@ class sfp_pgp(SpiderFootPlugin):
meta = {
'name': "PGP Key Servers",
'summary': "Look up e-mail addresses in PGP public key servers.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Public Registries" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Public Registries"]
}
results = None
diff --git a/modules/sfp_phishstats.py b/modules/sfp_phishstats.py
index cde22328..3ea27841 100644
--- a/modules/sfp_phishstats.py
+++ b/modules/sfp_phishstats.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
from netaddr import IPNetwork
import urllib.request, urllib.parse, urllib.error
import json
@@ -22,9 +22,9 @@ class sfp_phishstats(SpiderFootPlugin):
meta = {
'name': "PhishStats",
'summary': "Determine if an IP Address is malicious",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://phishstats.info/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -67,13 +67,21 @@ class sfp_phishstats(SpiderFootPlugin):
# What events is this module interested in for input
# For a list of all events, check sfdb.py.
def watchedEvents(self):
- return ["IP_ADDRESS", "NETBLOCK_OWNER", "NETBLOCK_MEMBER",
- "AFFILIATE_IPADDR"]
+ return [
+ "IP_ADDRESS",
+ "NETBLOCK_OWNER",
+ "NETBLOCK_MEMBER",
+ "AFFILIATE_IPADDR"
+ ]
# What events this module produces
def producedEvents(self):
- return ["IP_ADDRESS", "MALICIOUS_IPADDR", "RAW_RIR_DATA",
- "MALICIOUS_AFFILIATE_IPADDR"]
+ return [
+ "IP_ADDRESS",
+ "MALICIOUS_IPADDR",
+ "RAW_RIR_DATA",
+ "MALICIOUS_AFFILIATE_IPADDR"
+ ]
# Check whether the IP Address is malicious using Phishstats API
# https://phishstats.info/
@@ -84,7 +92,7 @@ class sfp_phishstats(SpiderFootPlugin):
}
headers = {
- 'Accept' : "application/json",
+ 'Accept': "application/json",
}
res = self.sf.fetchUrl(
@@ -126,22 +134,22 @@ class sfp_phishstats(SpiderFootPlugin):
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxnetblock']))
- return None
+
+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
+ self.sf.debug("Network size bigger than permitted: " +
+ str(IPNetwork(eventData).prefixlen) + " > " +
+ str(self.opts['maxnetblock']))
+ return None
if eventName == 'NETBLOCK_MEMBER':
if not self.opts['subnetlookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxsubnet']))
- return None
+
+ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:
+ self.sf.debug("Network size bigger than permitted: " +
+ str(IPNetwork(eventData).prefixlen) + " > " +
+ str(self.opts['maxsubnet']))
+ return None
qrylist = list()
if eventName.startswith("NETBLOCK_"):
diff --git a/modules/sfp_phishtank.py b/modules/sfp_phishtank.py
index c0db17d8..2b4639ae 100644
--- a/modules/sfp_phishtank.py
+++ b/modules/sfp_phishtank.py
@@ -13,14 +13,14 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'PhishTank': {
'id': '_phishtank',
'checks': ['domain'],
'url': 'http://data.phishtank.com/data/online-valid.csv',
- 'regex': '\d+,\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*,http://www.phishtank.com/.*'
+ 'regex': r'\d+,\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*,http://www.phishtank.com/.*'
}
}
@@ -31,9 +31,9 @@ class sfp_phishtank(SpiderFootPlugin):
meta = {
'name': "PhishTank",
'summary': "Check if a host/domain is malicious according to PhishTank.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://phishtank.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -117,8 +117,7 @@ class sfp_phishtank(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_phone.py b/modules/sfp_phone.py
index d6d11d67..e0c1d4cf 100644
--- a/modules/sfp_phone.py
+++ b/modules/sfp_phone.py
@@ -15,7 +15,7 @@
import phonenumbers
from phonenumbers import carrier
#from phonenumbers import geocoder
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_phone(SpiderFootPlugin):
"""Phone Number Extractor:Passive,Footprint,Investigate:Content Analysis::Identify phone numbers in scraped webpages."""
@@ -23,9 +23,9 @@ class sfp_phone(SpiderFootPlugin):
meta = {
'name': "Phone Number Extractor",
'summary': "Identify phone numbers in scraped webpages.",
- 'flags': [ "" ],
- 'useCases': [ "Passive", "Footprint", "Investigate" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Passive", "Footprint", "Investigate"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -70,8 +70,7 @@ class sfp_phone(SpiderFootPlugin):
content = eventData.replace('.', '-')
for match in phonenumbers.PhoneNumberMatcher(content, region=None):
- n = phonenumbers.format_number(match.number,
- phonenumbers.PhoneNumberFormat.E164)
+ n = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
evt = SpiderFootEvent("PHONE_NUMBER", n, self.__name__, event)
if event.moduleDataSource:
evt.moduleDataSource = event.moduleDataSource
diff --git a/modules/sfp_psbdmp.py b/modules/sfp_psbdmp.py
index 4869cce7..5f951125 100644
--- a/modules/sfp_psbdmp.py
+++ b/modules/sfp_psbdmp.py
@@ -12,7 +12,7 @@
import re
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_psbdmp(SpiderFootPlugin):
"""Psbdmp:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Check psbdmp.cc (PasteBin Dump) for potentially hacked e-mails and domains."""
@@ -20,9 +20,9 @@ class sfp_psbdmp(SpiderFootPlugin):
meta = {
'name': "Psbdmp",
'summary': "Check psbdmp.cc (PasteBin Dump) for potentially hacked e-mails and domains.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://psbdmp.cc/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -81,8 +81,7 @@ class sfp_psbdmp(SpiderFootPlugin):
else:
url = "https://psbdmp.cc/api/search/domain/" + qry
- res = self.sf.fetchUrl(url, timeout=15,
- useragent="SpiderFoot")
+ res = self.sf.fetchUrl(url, timeout=15, useragent="SpiderFoot")
if res['code'] == "403" or res['content'] is None:
self.sf.info("Unable to fetch data from psbdmp.cc right now.")
@@ -137,13 +136,13 @@ class sfp_psbdmp(SpiderFootPlugin):
continue
# Sometimes pastes search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
continue
try:
startIndex = res['content'].index(eventData)
- except BaseException as e:
+ except BaseException:
self.sf.debug("String not found in pastes content.")
continue
diff --git a/modules/sfp_pulsedive.py b/modules/sfp_pulsedive.py
index 60630c33..d7a5041f 100644
--- a/modules/sfp_pulsedive.py
+++ b/modules/sfp_pulsedive.py
@@ -15,7 +15,7 @@ from datetime import datetime
import time
from netaddr import IPNetwork
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_pulsedive(SpiderFootPlugin):
"""Pulsedive:Investigate,Passive:Reputation Systems:apikey:Obtain information from Pulsedive's API."""
@@ -23,9 +23,9 @@ class sfp_pulsedive(SpiderFootPlugin):
meta = {
'name': "Pulsedive",
'summary': "Obtain information from Pulsedive's API.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://pulsedive.com/",
'model': "FREE_AUTH_LIMITED",
@@ -119,7 +119,7 @@ class sfp_pulsedive(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from Pulsedive.", False)
+ self.sf.error(f"Error processing JSON response from Pulsedive: {e}", False)
return None
return info
@@ -226,7 +226,7 @@ class sfp_pulsedive(SpiderFootPlugin):
if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:
self.sf.debug("Record found but too old, skipping.")
continue
- except BaseException as e:
+ except BaseException:
self.sf.debug("Couldn't parse date from Pulsedive so assuming it's OK.")
e = SpiderFootEvent(evtType, descr, self.__name__, event)
self.notifyListeners(e)
diff --git a/modules/sfp_quad9.py b/modules/sfp_quad9.py
index a24ba473..099b5467 100644
--- a/modules/sfp_quad9.py
+++ b/modules/sfp_quad9.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_quad9(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_quad9(SpiderFootPlugin):
meta = {
'name': "Quad9",
'summary': "Check if a host would be blocked by Quad9",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://quad9.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -72,13 +72,13 @@ class sfp_quad9(SpiderFootPlugin):
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "9.9.9.9" ]
+ res.nameservers = ["9.9.9.9"]
try:
addrs = res.query(qaddr)
self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -104,7 +104,7 @@ class sfp_quad9(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
+ except BaseException:
return None
if resolved:
diff --git a/modules/sfp_riskiq.py b/modules/sfp_riskiq.py
index 684affd3..3ae5e1c0 100644
--- a/modules/sfp_riskiq.py
+++ b/modules/sfp_riskiq.py
@@ -12,7 +12,7 @@
import json
import base64
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_riskiq(SpiderFootPlugin):
"""RiskIQ:Investigate,Passive:Reputation Systems:apikey:Obtain information from RiskIQ's (formerly PassiveTotal) Passive DNS and Passive SSL databases."""
@@ -20,9 +20,9 @@ class sfp_riskiq(SpiderFootPlugin):
meta = {
'name': "RiskIQ",
'summary': "Obtain information from RiskIQ's (formerly PassiveTotal) Passive DNS and Passive SSL databases.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://community.riskiq.com/",
'model': "FREE_AUTH_LIMITED",
@@ -127,7 +127,7 @@ class sfp_riskiq(SpiderFootPlugin):
useragent="SpiderFoot", headers=headers,
postData=post)
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("RiskIQ access seems to have been rejected or you have exceeded usage limits.", False)
self.errorState = True
return None
@@ -142,7 +142,7 @@ class sfp_riskiq(SpiderFootPlugin):
self.sf.info("No RiskIQ info found for " + qry)
return None
except BaseException as e:
- self.sf.error("Invalid JSON returned by RiskIQ.", False)
+ self.sf.error(f"Invalid JSON returned by RiskIQ: {e}", False)
return None
return ret['results']
@@ -176,7 +176,7 @@ class sfp_riskiq(SpiderFootPlugin):
else:
self.results[eventData] = True
- if eventName in [ 'DOMAIN_NAME' ]:
+ if eventName in ['DOMAIN_NAME']:
ret = self.query(eventData, "PSSL")
if not ret:
self.sf.info("No RiskIQ passive SSL data found for " + eventData)
@@ -220,7 +220,7 @@ class sfp_riskiq(SpiderFootPlugin):
return None
- if eventName in [ 'IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME' ]:
+ if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']:
ret = self.query(eventData, "PDNS")
if not ret:
self.sf.info("No RiskIQ passive DNS data found for " + eventData)
@@ -237,7 +237,7 @@ class sfp_riskiq(SpiderFootPlugin):
# We found a co-host
cohosts.append(r['focusPoint'])
- if eventName in [ "INTERNET_NAME", "DOMAIN_NAME" ]:
+ if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]:
# Record could be an A/CNAME of this entity, or something pointing to it
for r in ret:
if r['focusPoint'].endswith("."):
diff --git a/modules/sfp_robtex.py b/modules/sfp_robtex.py
index e6f3318c..a0c5fea8 100644
--- a/modules/sfp_robtex.py
+++ b/modules/sfp_robtex.py
@@ -13,7 +13,7 @@
import json
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_robtex(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_robtex(SpiderFootPlugin):
meta = {
'name': "Robtex",
'summary': "Search Robtex.com for hosts sharing the same IP.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "https://www.robtex.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -85,7 +85,7 @@ class sfp_robtex(SpiderFootPlugin):
# Don't notify me about events from myself
def watchOpts(self):
- return [ 'noself' ]
+ return ['noself']
# Handle events sent to this module
def handleEvent(self, event):
@@ -137,16 +137,13 @@ class sfp_robtex(SpiderFootPlugin):
qrylist.append(eventData)
self.results[eventData] = True
- myres = list()
-
for ip in qrylist:
if self.checkForStop():
return None
retry = 0
while retry < 2:
- res = self.sf.fetchUrl("https://freeapi.robtex.com/ipquery/" + ip,
- timeout=self.opts['_fetchtimeout'])
+ res = self.sf.fetchUrl("https://freeapi.robtex.com/ipquery/" + ip, timeout=self.opts['_fetchtimeout'])
if res['code'] == "200":
break
if res['code'] == "404":
@@ -164,7 +161,7 @@ class sfp_robtex(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except BaseException as e:
- self.sf.error("Error parsing JSON from robtex API.", False)
+ self.sf.error(f"Error parsing JSON from Robtex API: {e}", False)
return None
pas = data.get('pas')
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
index ca441d21..5fb4e8ec 100644
--- a/modules/sfp_s3bucket.py
+++ b/modules/sfp_s3bucket.py
@@ -13,7 +13,7 @@
import threading
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_s3bucket(SpiderFootPlugin):
"""Amazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents."""
@@ -21,9 +21,9 @@ class sfp_s3bucket(SpiderFootPlugin):
meta = {
'name': "Amazon S3 Bucket Finder",
'summary': "Search for potential Amazon S3 buckets associated with the target and attempt to list their contents.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Crawling and Scanning" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Crawling and Scanning"],
'dataSource': {
'website': "https://aws.amazon.com/s3/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -70,7 +70,7 @@ class sfp_s3bucket(SpiderFootPlugin):
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
- if res['code'] not in [ "301", "302", "200" ] and \
+ if res['code'] not in ["301", "302", "200"] and \
(res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
@@ -82,7 +82,6 @@ class sfp_s3bucket(SpiderFootPlugin):
self.s3results[url] = 0
def threadSites(self, siteList):
- ret = list()
self.s3results = dict()
running = True
i = 0
@@ -156,7 +155,7 @@ class sfp_s3bucket(SpiderFootPlugin):
self.notifyListeners(evt)
return None
- targets = [ eventData.replace('.', '') ]
+ targets = [eventData.replace('.', '')]
kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
if kw:
targets.append(kw)
diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.py
index 1be9a0f7..d212ac73 100644
--- a/modules/sfp_scylla.py
+++ b/modules/sfp_scylla.py
@@ -13,7 +13,7 @@ import json
import time
import base64
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_scylla(SpiderFootPlugin):
"""Scylla:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Gather breach data from Scylla API."""
@@ -21,9 +21,9 @@ class sfp_scylla(SpiderFootPlugin):
meta = {
'name': "Scylla",
'summary': "Gather breach data from Scylla API.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://scylla.sh/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -47,8 +47,8 @@ class sfp_scylla(SpiderFootPlugin):
# Option descriptions
optdescs = {
- 'pause': "Number of seconds to pause between fetches.",
- 'per_page': "Maximum number of results per page.",
+ 'pause': "Number of seconds to pause between fetches.",
+ 'per_page': "Maximum number of results per page.",
'max_pages': "Maximum number of pages of results to fetch."
}
@@ -65,11 +65,11 @@ class sfp_scylla(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ 'DOMAIN_NAME' ]
+ return ['DOMAIN_NAME']
# What events this module produces
def producedEvents(self):
- return [ 'EMAILADDR_COMPROMISED', 'PASSWORD_COMPROMISED', 'HASH_COMPROMISED', 'RAW_RIR_DATA' ]
+ return ['EMAILADDR_COMPROMISED', 'PASSWORD_COMPROMISED', 'HASH_COMPROMISED', 'RAW_RIR_DATA']
# Query Scylla API
def query(self, qry, per_page=20, start=0):
diff --git a/modules/sfp_securitytrails.py b/modules/sfp_securitytrails.py
index 8d69529b..6836b01f 100644
--- a/modules/sfp_securitytrails.py
+++ b/modules/sfp_securitytrails.py
@@ -12,7 +12,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_securitytrails(SpiderFootPlugin):
"""SecurityTrails:Investigate,Passive:Search Engines:apikey:Obtain Passive DNS and other information from SecurityTrails"""
@@ -20,9 +20,9 @@ class sfp_securitytrails(SpiderFootPlugin):
meta = {
'name': "SecurityTrails",
'summary': "Obtain Passive DNS and other information from SecurityTrails",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://securitytrails.com/",
'model': "FREE_AUTH_LIMITED",
@@ -111,7 +111,7 @@ class sfp_securitytrails(SpiderFootPlugin):
useragent="SpiderFoot", headers=headers,
postData=request)
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("SecurityTrails API key seems to have been rejected or you have exceeded usage limits for the month.", False)
self.errorState = True
return None
@@ -166,7 +166,7 @@ class sfp_securitytrails(SpiderFootPlugin):
else:
self.results[eventData] = True
- if eventName in [ "IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]:
+ if eventName in ["IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]:
ip = eventData
rec = self.query(ip, "ipv4")
myres = list()
@@ -200,7 +200,7 @@ class sfp_securitytrails(SpiderFootPlugin):
self.notifyListeners(e)
self.cohostcount += 1
- if eventName in [ "EMAILADDR"]:
+ if eventName in ["EMAILADDR"]:
email = eventData
rec = self.query(email, "whois_email")
myres = list()
diff --git a/modules/sfp_shodan.py b/modules/sfp_shodan.py
index 9a257b36..2085a0c1 100644
--- a/modules/sfp_shodan.py
+++ b/modules/sfp_shodan.py
@@ -14,7 +14,7 @@ import json
import urllib.request, urllib.parse, urllib.error
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_shodan(SpiderFootPlugin):
@@ -23,9 +23,9 @@ class sfp_shodan(SpiderFootPlugin):
meta = {
'name': "SHODAN",
'summary': "Obtain information from SHODAN about identified IP addresses.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.shodan.io/",
'model': "FREE_AUTH_LIMITED",
@@ -97,7 +97,7 @@ class sfp_shodan(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from SHODAN.", False)
+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)
return None
return info
@@ -114,7 +114,7 @@ class sfp_shodan(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from SHODAN.", False)
+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)
return None
return info
@@ -135,7 +135,7 @@ class sfp_shodan(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from SHODAN.", False)
+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)
return None
return info
diff --git a/modules/sfp_similar.py b/modules/sfp_similar.py
index e90c4666..f057b396 100644
--- a/modules/sfp_similar.py
+++ b/modules/sfp_similar.py
@@ -11,50 +11,50 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
nearchars = {
- 'a': [ '4', 's' ],
- 'b': [ 'v', 'n' ],
- 'c': [ 'x', 'v' ],
- 'd': [ 's', 'f' ],
- 'e': [ 'w', 'r' ],
- 'f': [ 'd', 'g' ],
- 'g': [ 'f', 'h' ],
- 'h': [ 'g', 'j', 'n' ],
- 'i': [ 'o', 'u', '1' ],
- 'j': [ 'k', 'h', 'i' ],
- 'k': [ 'l', 'j' ],
- 'l': [ 'i', '1', 'k' ],
- 'm': [ 'n' ],
- 'n': [ 'm' ],
- 'o': [ 'p', 'i', '0' ],
- 'p': [ 'o', 'q' ],
- 'r': [ 't', 'e' ],
- 's': [ 'a', 'd', '5' ],
- 't': [ '7', 'y', 'z', 'r' ],
- 'u': [ 'v', 'i', 'y', 'z' ],
- 'v': [ 'u', 'c', 'b' ],
- 'w': [ 'v', 'vv', 'q', 'e' ],
- 'x': [ 'z', 'y', 'c' ],
- 'y': [ 'z', 'x' ],
- 'z': [ 'y', 'x' ],
- '0': [ 'o' ],
- '1': [ 'l' ],
- '2': [ '5' ],
- '3': [ 'e' ],
- '4': [ 'a' ],
- '5': [ 's' ],
- '6': [ 'b' ],
- '7': [ 't' ],
- '8': [ 'b' ],
+ 'a': ['4', 's'],
+ 'b': ['v', 'n'],
+ 'c': ['x', 'v'],
+ 'd': ['s', 'f'],
+ 'e': ['w', 'r'],
+ 'f': ['d', 'g'],
+ 'g': ['f', 'h'],
+ 'h': ['g', 'j', 'n'],
+ 'i': ['o', 'u', '1'],
+ 'j': ['k', 'h', 'i'],
+ 'k': ['l', 'j'],
+ 'l': ['i', '1', 'k'],
+ 'm': ['n'],
+ 'n': ['m'],
+ 'o': ['p', 'i', '0'],
+ 'p': ['o', 'q'],
+ 'r': ['t', 'e'],
+ 's': ['a', 'd', '5'],
+ 't': ['7', 'y', 'z', 'r'],
+ 'u': ['v', 'i', 'y', 'z'],
+ 'v': ['u', 'c', 'b'],
+ 'w': ['v', 'vv', 'q', 'e'],
+ 'x': ['z', 'y', 'c'],
+ 'y': ['z', 'x'],
+ 'z': ['y', 'x'],
+ '0': ['o'],
+ '1': ['l'],
+ '2': ['5'],
+ '3': ['e'],
+ '4': ['a'],
+ '5': ['s'],
+ '6': ['b'],
+ '7': ['t'],
+ '8': ['b'],
'9': []
}
pairs = {
- 'oo': [ '00' ],
- 'll': [ 'l1l', 'l1l', '111', '11' ],
- '11': [ 'll', 'lll', 'l1l', '1l1' ]
+ 'oo': ['00'],
+ 'll': ['l1l', 'l1l', '111', '11'],
+ '11': ['ll', 'lll', 'l1l', '1l1']
}
# These are all string builders, {0} = the domain keyword, {1} = the page number
@@ -64,9 +64,9 @@ class sfp_similar(SpiderFootPlugin):
meta = {
'name': "Similar Domain Finder",
'summary': "Search various sources to identify similar looking domain names, for instance squatted domains.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "DNS" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["DNS"]
}
# Default options
@@ -100,8 +100,6 @@ class sfp_similar(SpiderFootPlugin):
# Search for similar sounding domains
def handleEvent(self, event):
- eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
domlist = list()
@@ -160,7 +158,7 @@ class sfp_similar(SpiderFootPlugin):
self.sf.debug("Resolved " + d + tld)
evt = SpiderFootEvent("SIMILARDOMAIN", d + tld, self.__name__, event)
self.notifyListeners(evt)
- except BaseException as e:
+ except BaseException:
continue
return None
diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.py
index c4caf2de..7ddefa89 100644
--- a/modules/sfp_skymem.py
+++ b/modules/sfp_skymem.py
@@ -13,7 +13,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_skymem(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_skymem(SpiderFootPlugin):
meta = {
'name': "Skymem",
'summary': "Look up e-mail addresses on Skymem.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "http://www.skymem.info/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -138,7 +138,7 @@ class sfp_skymem(SpiderFootPlugin):
# Check if we're on the last page of results
max_page = 0
- pages = re.findall(r'/domain/' + domain_id + '\?p=(\d+)', res['content'])
+ pages = re.findall(r'/domain/' + domain_id + r'\?p=(\d+)', res['content'])
for p in pages:
if int(p) >= max_page:
max_page = int(p)
diff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.py
index f818ad21..67f4ea03 100644
--- a/modules/sfp_slideshare.py
+++ b/modules/sfp_slideshare.py
@@ -11,7 +11,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_slideshare(SpiderFootPlugin):
"""SlideShare:Footprint,Investigate,Passive:Social Media::Gather name and location from SlideShare profiles."""
@@ -19,9 +19,9 @@ class sfp_slideshare(SpiderFootPlugin):
meta = {
'name': "SlideShare",
'summary': "Gather name and location from SlideShare profiles.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://www.slideshare.net",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -57,15 +57,15 @@ class sfp_slideshare(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "SOCIAL_MEDIA" ]
+ return ["SOCIAL_MEDIA"]
# What events this module produces
def producedEvents(self):
- return [ "RAW_RIR_DATA", "GEOINFO" ]
+ return ["RAW_RIR_DATA", "GEOINFO"]
# Extract meta property contents from HTML
def extractMeta(self, meta_property, html):
- return re.findall(r'<meta property="' + meta_property + '"\s+content="(.+)" />', html)
+ return re.findall(r'<meta property="' + meta_property + r'"\s+content="(.+)" />', html)
# Handle events sent to this module
def handleEvent(self, event):
diff --git a/modules/sfp_snov.py b/modules/sfp_snov.py
index 0d1c5d63..67fb79fa 100644
--- a/modules/sfp_snov.py
+++ b/modules/sfp_snov.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import urllib.request, urllib.parse, urllib.error
import json
@@ -21,9 +21,9 @@ class sfp_snov(SpiderFootPlugin):
meta = {
'name': "Snov",
'summary': "Gather available email IDs from identified domains",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://snov.io/",
'model': "FREE_AUTH_LIMITED",
@@ -127,7 +127,7 @@ class sfp_snov(SpiderFootPlugin):
}
headers = {
- 'Accept' : "application/json",
+ 'Accept': "application/json",
}
res = self.sf.fetchUrl(
diff --git a/modules/sfp_social.py b/modules/sfp_social.py
index 6f29f94a..1f538d93 100644
--- a/modules/sfp_social.py
+++ b/modules/sfp_social.py
@@ -12,21 +12,21 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
regexps = dict({
"LinkedIn (Individual)": list(['.*linkedin.com/in/([a-zA-Z0-9_]+$)']),
"LinkedIn (Company)": list(['.*linkedin.com/company/([a-zA-Z0-9_]+$)']),
- "Github": list(['.*github.com/([a-zA-Z0-9_]+)\/']),
+ "Github": list([r'.*github.com/([a-zA-Z0-9_]+)\/']),
"Google+": list(['.*plus.google.com/([0-9]+$)']),
"Facebook": list(['.*facebook.com/([a-zA-Z0-9_]+$)']),
- "MySpace": list(['https?://myspace.com/([a-zA-Z0-9_\.]+$)']),
+ "MySpace": list([r'https?://myspace.com/([a-zA-Z0-9_\.]+$)']),
"YouTube": list(['.*youtube.com/([a-zA-Z0-9_]+$)']),
"Twitter": list(['.*twitter.com/([a-zA-Z0-9_]{1,15}$)',
'.*twitter.com/#!/([a-zA-Z0-9_]{1,15}$)'
]),
"SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)']),
- "Instagram": list(['.*instagram.com/([a-zA-Z0-9_\.]+)/?$'])
+ "Instagram": list([r'.*instagram.com/([a-zA-Z0-9_\.]+)/?$'])
})
@@ -36,9 +36,9 @@ class sfp_social(SpiderFootPlugin):
meta = {
'name': "Social Network Identifier",
'summary': "Identify presence on social media networks such as LinkedIn, Twitter and others.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Social Media" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Social Media"]
}
# Default options
@@ -98,7 +98,7 @@ class sfp_social(SpiderFootPlugin):
# Except for Google+, the link includes potential usernames
if regexpGrp != "Google+":
un = bits.group(1)
- evt = SpiderFootEvent("USERNAME", bits.group(1), self.__name__, event)
+ evt = SpiderFootEvent("USERNAME", str(un), self.__name__, event)
self.notifyListeners(evt)
return None
diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.py
index fe705eae..e1238c9a 100644
--- a/modules/sfp_socialprofiles.py
+++ b/modules/sfp_socialprofiles.py
@@ -13,7 +13,7 @@
import re
import urllib.request, urllib.error, urllib.parse
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
sites = {
# Search string to use, domain name the profile will sit on within
@@ -21,22 +21,22 @@ sites = {
"Facebook": [
"\"{name}\"+site:facebook.com",
[
- '[ \'"](https?://[a-z\.]*facebook.[a-z\.]+/[^/"\'<> ]+/?)[\'" ]',
- '(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/"\'<> ]+/?)',
+ r'[ \'"](https?://[a-z\.]*facebook.[a-z\.]+/[^/"\'<> ]+/?)[\'"]',
+ r'(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/"\'<> ]+/?)',
],
],
"Google+": [
"\"{name}\"+site:plus.google.com",
[
- '[ \'"](https?://plus.google.[a-z\.]+/\d+[^"\'<>\/ ]+)[\'" ]',
- '(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/"\'<> ]+)',
+ r'[ \'"](https?://plus.google.[a-z\.]+/\d+[^"\'<>\/ ]+)[\'"]',
+ r'(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/"\'<> ]+)',
],
],
"LinkedIn": [
"\"{name}\"+site:linkedin.com",
[
- '["\' ](https?://[a-z\.]*linkedin.[a-z\.]+/[^\?"\'<> ]+)[\'" ]',
- '(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\?"\'<> ]+)',
+ r'["\' ](https?://[a-z\.]*linkedin.[a-z\.]+/[^\?"\'<> ]+)[\'"]',
+ r'(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\?"\'<> ]+)',
],
],
}
@@ -48,9 +48,9 @@ class sfp_socialprofiles(SpiderFootPlugin):
meta = {
'name': "Social Media Profile Finder",
'summary': "Tries to discover the social media profiles for human names identified.",
- 'flags': [ "slow", "apikey" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Social Media" ]
+ 'flags': ["slow", "apikey"],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Social Media"]
}
# Default options
@@ -219,7 +219,7 @@ class sfp_socialprofiles(SpiderFootPlugin):
found = False
for kw in self.keywords:
if re.search(
- "[^a-zA-Z\-\_]" + kw + "[^a-zA-Z\-\_]",
+ r"[^a-zA-Z\-\_]" + kw + r"[^a-zA-Z\-\_]",
pres["content"],
re.IGNORECASE,
):
diff --git a/modules/sfp_sorbs.py b/modules/sfp_sorbs.py
index dc5aa493..0e25388b 100644
--- a/modules/sfp_sorbs.py
+++ b/modules/sfp_sorbs.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_sorbs(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_sorbs(SpiderFootPlugin):
meta = {
'name': "SORBS",
'summary': "Query the SORBS database for open relays, open proxies, vulnerable servers, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.sorbs.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -154,7 +154,6 @@ class sfp_sorbs(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
parentEvent = event
- addrlist = list()
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
diff --git a/modules/sfp_spamcop.py b/modules/sfp_spamcop.py
index a49d0817..dc95b454 100644
--- a/modules/sfp_spamcop.py
+++ b/modules/sfp_spamcop.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_spamcop(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_spamcop(SpiderFootPlugin):
meta = {
'name': "SpamCop",
'summary': "Query various spamcop databases for open relays, open proxies, vulnerable servers, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.spamcop.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -145,7 +145,6 @@ class sfp_spamcop(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
parentEvent = event
- addrlist = list()
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
diff --git a/modules/sfp_spamhaus.py b/modules/sfp_spamhaus.py
index 0a4b5fae..3a94befd 100644
--- a/modules/sfp_spamhaus.py
+++ b/modules/sfp_spamhaus.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_spamhaus(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_spamhaus(SpiderFootPlugin):
meta = {
'name': "Spamhaus",
'summary': "Query the Spamhaus databases for open relays, open proxies, vulnerable servers, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.spamhaus.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -157,7 +157,6 @@ class sfp_spamhaus(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
parentEvent = event
- addrlist = list()
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 493b5e41..3a48b756 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -13,7 +13,7 @@
import time
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_spider(SpiderFootPlugin):
"""Web Spider:Footprint,Investigate:Crawling and Scanning:slow:Spidering of web-pages to extract content for searching."""
@@ -21,9 +21,9 @@ class sfp_spider(SpiderFootPlugin):
meta = {
'name': "Web Spider",
'summary': "Spidering of web-pages to extract content for searching.",
- 'flags': [ "slow" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Crawling and Scanning" ]
+ 'flags': ["slow"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Crawling and Scanning"]
}
# Default options
@@ -250,7 +250,7 @@ class sfp_spider(SpiderFootPlugin):
# Don't notify me about events from myself
def watchOpts(self):
- return [ 'noself' ]
+ return ['noself']
# What events this module produces
# This is to support the end user in selecting modules based on events
diff --git a/modules/sfp_spur.py b/modules/sfp_spur.py
index c6992c86..539fa26a 100644
--- a/modules/sfp_spur.py
+++ b/modules/sfp_spur.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
from netaddr import IPNetwork
import json
@@ -21,9 +21,9 @@ class sfp_spur(SpiderFootPlugin):
meta = {
'name': "spur.us",
'summary': "Obtain information about any malicious activities involving IP addresses found",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://spur.us/",
'model': "COMMERCIAL ONLY",
@@ -78,13 +78,23 @@ class sfp_spur(SpiderFootPlugin):
# What events is this module interested in for input
# For a list of all events, check sfdb.py.
def watchedEvents(self):
- return ["IP_ADDRESS", "NETBLOCK_OWNER", "NETBLOCK_MEMBER",
- "AFFILIATE_IPADDR"]
+ return [
+ "IP_ADDRESS",
+ "NETBLOCK_OWNER",
+ "NETBLOCK_MEMBER",
+ "AFFILIATE_IPADDR"
+ ]
# What events this module produces
def producedEvents(self):
- return ["IP_ADDRESS", "MALICIOUS_IPADDR", "RAW_RIR_DATA",
- "GEO_INFO", "COMPANY_NAME", "MALICIOUS_AFFILIATE_IPADDR"]
+ return [
+ "IP_ADDRESS",
+ "MALICIOUS_IPADDR",
+ "RAW_RIR_DATA",
+ "GEO_INFO",
+ "COMPANY_NAME",
+ "MALICIOUS_AFFILIATE_IPADDR"
+ ]
# Check whether the IP Address is malicious using spur.us API
# https://spur.us/app/docs
@@ -143,29 +153,28 @@ class sfp_spur(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug(f"Skipping {eventData}, already checked.")
return None
- else:
- self.results[eventData] = True
+ self.results[eventData] = True
if eventName == 'NETBLOCK_OWNER':
if not self.opts['netblocklookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxnetblock']))
- return None
+
+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
+ self.sf.debug("Network size bigger than permitted: " +
+ str(IPNetwork(eventData).prefixlen) + " > " +
+ str(self.opts['maxnetblock']))
+ return None
if eventName == 'NETBLOCK_MEMBER':
if not self.opts['subnetlookup']:
return None
- else:
- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:
- self.sf.debug("Network size bigger than permitted: " +
- str(IPNetwork(eventData).prefixlen) + " > " +
- str(self.opts['maxsubnet']))
- return None
+
+ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:
+ self.sf.debug("Network size bigger than permitted: " +
+ str(IPNetwork(eventData).prefixlen) + " > " +
+ str(self.opts['maxsubnet']))
+ return None
qrylist = list()
if eventName.startswith("NETBLOCK_"):
@@ -250,7 +259,7 @@ class sfp_spur(SpiderFootPlugin):
if vpnOperatorsExists:
vpnOperatorNames = vpnOperators.get('operators')
- maliciousIPDesc = "spur.us [ " + str(addr) + " ]\n"
+ maliciousIPDesc = "spur.us [" + str(addr) + "]\n"
maliciousIPDesc += "VPN Operators : "
for operatorNameDict in vpnOperatorNames:
diff --git a/modules/sfp_spyonweb.py b/modules/sfp_spyonweb.py
index 54ce0413..c470abc6 100644
--- a/modules/sfp_spyonweb.py
+++ b/modules/sfp_spyonweb.py
@@ -14,7 +14,7 @@
import datetime
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_spyonweb(SpiderFootPlugin):
"""SpyOnWeb:Footprint,Investigate,Passive:Passive DNS:apikey:Search SpyOnWeb for hosts sharing the same IP address, Google Analytics code, or Google Adsense code."""
@@ -22,9 +22,9 @@ class sfp_spyonweb(SpiderFootPlugin):
meta = {
'name': "SpyOnWeb",
'summary': "Search SpyOnWeb for hosts sharing the same IP address, Google Analytics code, or Google Adsense code.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "http://spyonweb.com/",
'model': "FREE_AUTH_LIMITED",
@@ -105,7 +105,7 @@ class sfp_spyonweb(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
status = data.get('status')
@@ -212,7 +212,7 @@ class sfp_spyonweb(SpiderFootPlugin):
agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])
# Find Google AdSense IDs and Google Analytics IDs for the specified domain
- if eventName in [ 'INTERNET_NAME', 'DOMAIN_NAME' ]:
+ if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:
data = self.querySummary(eventData, limit=self.opts['limit'])
if data is None:
@@ -234,7 +234,7 @@ class sfp_spyonweb(SpiderFootPlugin):
self.notifyListeners(evt)
# Find affiliate domains for the specified Google AdSense ID or Google Analytics ID
- if eventName in [ 'WEB_ANALYTICS_ID' ]:
+ if eventName in ['WEB_ANALYTICS_ID']:
try:
network = eventData.split(": ")[0]
analytics_id = eventData.split(": ")[1]
@@ -270,14 +270,14 @@ class sfp_spyonweb(SpiderFootPlugin):
self.notifyListeners(evt)
# Find co-hosts on the same IP address
- if eventName in [ 'IP_ADDRESS' ]:
+ if eventName in ['IP_ADDRESS']:
data = self.queryIP(eventData, limit=self.opts['limit'])
if data is None:
self.sf.info("No data found for " + eventData)
return None
- cohostcount = 0
+ self.cohostcount = 0
for co in list(data.keys()):
last_seen = int(datetime.datetime.strptime(data[co], '%Y-%m-%d').strftime('%s')) * 1000
diff --git a/modules/sfp_spyse.py b/modules/sfp_spyse.py
index 3423accc..9bb5c790 100644
--- a/modules/sfp_spyse.py
+++ b/modules/sfp_spyse.py
@@ -15,7 +15,7 @@
import json
import time
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_spyse(SpiderFootPlugin):
"""Spyse:Footprint,Investigate,Passive:Passive DNS:apikey:SpiderFoot plug-in to search Spyse API for IP address and domain information."""
@@ -23,9 +23,9 @@ class sfp_spyse(SpiderFootPlugin):
meta = {
'name': "Spyse",
'summary': "SpiderFoot plug-in to search Spyse API for IP address and domain information.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Passive DNS" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"],
'dataSource': {
'website': "https://spyse.com",
'model': "FREE_AUTH_LIMITED",
@@ -102,8 +102,8 @@ class sfp_spyse(SpiderFootPlugin):
'offset': currentOffset
}
headers = {
- 'Accept' : "application/json",
- 'Authorization' : "Bearer " + self.opts['api_key']
+ 'Accept': "application/json",
+ 'Authorization': "Bearer " + self.opts['api_key']
}
res = self.sf.fetchUrl(
@@ -126,8 +126,8 @@ class sfp_spyse(SpiderFootPlugin):
'offset': currentOffset
}
headers = {
- 'Accept' : "application/json",
- 'Authorization' : "Bearer " + self.opts['api_key']
+ 'Accept': "application/json",
+ 'Authorization': "Bearer " + self.opts['api_key']
}
res = self.sf.fetchUrl(
'https://api.spyse.com/v3/data/ip/port?' + urllib.parse.urlencode(params),
@@ -149,8 +149,8 @@ class sfp_spyse(SpiderFootPlugin):
'offset': currentOffset
}
headers = {
- 'Accept' : "application/json",
- 'Authorization' : "Bearer " + self.opts['api_key']
+ 'Accept': "application/json",
+ 'Authorization': "Bearer " + self.opts['api_key']
}
res = self.sf.fetchUrl(
'https://api.spyse.com/v3/data/ip/domain?' + urllib.parse.urlencode(params),
@@ -174,8 +174,8 @@ class sfp_spyse(SpiderFootPlugin):
}
headers = {
- 'Accept' : "application/json",
- 'Authorization' : "Bearer " + self.opts['api_key']
+ 'Accept': "application/json",
+ 'Authorization': "Bearer " + self.opts['api_key']
}
res = self.sf.fetchUrl(
@@ -224,7 +224,7 @@ class sfp_spyse(SpiderFootPlugin):
try:
data = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
if data.get('message'):
diff --git a/modules/sfp_sslcert.py b/modules/sfp_sslcert.py
index 81a0d754..bb156587 100644
--- a/modules/sfp_sslcert.py
+++ b/modules/sfp_sslcert.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
from urllib.parse import urlparse
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_sslcert(SpiderFootPlugin):
@@ -20,9 +20,9 @@ class sfp_sslcert(SpiderFootPlugin):
meta = {
'name': "SSL Certificate Analyzer",
'summary': "Gather information about SSL certificates used by the target's HTTPS sites.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Crawling and Scanning" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Crawling and Scanning"]
}
# Default options
@@ -91,7 +91,7 @@ class sfp_sslcert(SpiderFootPlugin):
if u.port:
port = u.port
fqdn = self.sf.urlFQDN(eventData.lower())
- except BaseException as e:
+ except BaseException:
self.sf.debug("Couldn't parse URL: " + eventData)
return None
else:
diff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.py
index 62149df8..66ae8d52 100644
--- a/modules/sfp_strangeheaders.py
+++ b/modules/sfp_strangeheaders.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
# Standard headers, taken from http://en.wikipedia.org/wiki/List_of_HTTP_header_fields
headers = ["access-control-allow-origin", "accept-ranges", "age", "allow", "cache-control",
@@ -30,9 +30,9 @@ class sfp_strangeheaders(SpiderFootPlugin):
meta = {
'name': "Strange Header Identifier",
'summary': "Obtain non-standard HTTP headers returned by web servers.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -80,7 +80,7 @@ class sfp_strangeheaders(SpiderFootPlugin):
jdata = json.loads(eventData)
if jdata == None:
return None
- except BaseException as e:
+ except BaseException:
self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
return None
diff --git a/modules/sfp_subdomain_takeover.py b/modules/sfp_subdomain_takeover.py
index ec7da1b3..1bb1b549 100644
--- a/modules/sfp_subdomain_takeover.py
+++ b/modules/sfp_subdomain_takeover.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_subdomain_takeover(SpiderFootPlugin):
"""Subdomain Takeover:Footprint,Investigate:Crawling and Scanning::Check if affiliated subdomains are vulnerable to takeover."""
@@ -21,9 +21,9 @@ class sfp_subdomain_takeover(SpiderFootPlugin):
meta = {
'name': "Subdomain Takeover",
'summary': "Check if affiliated subdomains are vulnerable to takeover.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Crawling and Scanning" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Crawling and Scanning"]
}
# Default options
@@ -63,7 +63,7 @@ class sfp_subdomain_takeover(SpiderFootPlugin):
try:
self.fingerprints = json.loads(content)
except Exception as e:
- self.sf.error("Unable to parse subdomain takeover fingerprints list.", False)
+ self.sf.error(f"Unable to parse subdomain takeover fingerprints list: {e}", False)
self.errorState = True
return None
diff --git a/modules/sfp_talosintel.py b/modules/sfp_talosintel.py
index a16ea71e..30f34905 100644
--- a/modules/sfp_talosintel.py
+++ b/modules/sfp_talosintel.py
@@ -13,7 +13,7 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'talosintelligence.com List': {
@@ -30,9 +30,9 @@ class sfp_talosintel(SpiderFootPlugin):
meta = {
'name': "Talos Intelligence",
'summary': "Check if a netblock or IP is malicious according to talosintelligence.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://talosintelligence.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -126,8 +126,7 @@ class sfp_talosintel(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_template.py b/modules/sfp_template.py
index 8493088b..26502215 100644
--- a/modules/sfp_template.py
+++ b/modules/sfp_template.py
@@ -62,13 +62,13 @@ class sfp_template(SpiderFootPlugin):
# - errorprone: Might generate high false positives
# - invasive: Interrogates the target, might be intensive
# - tool: Runs an external tool to collect data
- 'flags': [ "slow", "apikey"],
+ 'flags': ["slow", "apikey"],
# Use cases: The use case(s) this module should be included in, options are Footprint, Investigate and Passive.
# - Passive means the user's scan target is not contacted at all
# - Footprint means that this module is useful when understanding the target's footprint on the Internet
# - Investigate means that this module is useful when investigating the danger/risk of a target
- 'useCases': [ "Passive" ],
+ 'useCases': ["Passive"],
# Categories: The categories this module belongs in, describing how it operates. Only the first category is
# used for now.
@@ -83,7 +83,7 @@ class sfp_template(SpiderFootPlugin):
# - Search Engines: Searches public search engines with data about the whole Internet
# - Secondary Networks: Queries information about participation on secondary networks, like Bitcoin
# - Social Media: Searches social media data sources
- 'categories': [ "Social Media" ],
+ 'categories': ["Social Media"],
# For tool modules, have some basic information about the tool.
'toolDetails': {
diff --git a/modules/sfp_threatcrowd.py b/modules/sfp_threatcrowd.py
index 59fb8be0..e8f55eba 100644
--- a/modules/sfp_threatcrowd.py
+++ b/modules/sfp_threatcrowd.py
@@ -12,7 +12,7 @@
import json
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_threatcrowd(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_threatcrowd(SpiderFootPlugin):
meta = {
'name': "ThreatCrowd",
'summary': "Obtain information from ThreatCrowd about identified IP addresses, domains and e-mail addresses.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.threatcrowd.org",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -110,7 +110,7 @@ class sfp_threatcrowd(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from ThreatCrowd.", False)
+ self.sf.error(f"Error processing JSON response from ThreatCrowd: {e}", False)
self.errorState = True
return None
diff --git a/modules/sfp_threatminer.py b/modules/sfp_threatminer.py
index dee93aba..768ecc4b 100644
--- a/modules/sfp_threatminer.py
+++ b/modules/sfp_threatminer.py
@@ -14,7 +14,7 @@ import json
from datetime import datetime
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_threatminer(SpiderFootPlugin):
"""ThreatMiner:Footprint,Investigate,Passive:Search Engines::Obtain information from ThreatMiner's database for passive DNS and threat intelligence."""
@@ -22,9 +22,9 @@ class sfp_threatminer(SpiderFootPlugin):
meta = {
'name': "ThreatMiner",
'summary': "Obtain information from ThreatMiner's database for passive DNS and threat intelligence.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.threatminer.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -119,7 +119,7 @@ class sfp_threatminer(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from ThreatMiner.", False)
+ self.sf.error(f"Error processing JSON response from ThreatMiner: {e}", False)
return None
return info
diff --git a/modules/sfp_tldsearch.py b/modules/sfp_tldsearch.py
index 8c31005c..a1fd1259 100644
--- a/modules/sfp_tldsearch.py
+++ b/modules/sfp_tldsearch.py
@@ -15,7 +15,7 @@ import random
import threading
import time
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_tldsearch(SpiderFootPlugin):
"""TLD Searcher:Footprint:DNS:slow:Search all Internet TLDs for domains with the same name as the target (this can be very slow.)"""
@@ -23,9 +23,9 @@ class sfp_tldsearch(SpiderFootPlugin):
meta = {
'name': "TLD Searcher",
'summary': "Search all Internet TLDs for domains with the same name as the target (this can be very slow.)",
- 'flags': [ "slow" ],
- 'useCases': [ "Footprint" ],
- 'categories': [ "DNS" ]
+ 'flags': ["slow"],
+ 'useCases': ["Footprint"],
+ 'categories': ["DNS"]
}
# Default options
@@ -87,7 +87,7 @@ class sfp_tldsearch(SpiderFootPlugin):
else:
with self.lock:
self.tldResults[target] = True
- except BaseException as e:
+ except BaseException:
with self.lock:
self.tldResults[target] = False
@@ -146,8 +146,6 @@ class sfp_tldsearch(SpiderFootPlugin):
# Search for similar sounding domains
def handleEvent(self, event):
- eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if eventData in self.results:
diff --git a/modules/sfp_tool_cmseek.py b/modules/sfp_tool_cmseek.py
index 9ac03bc7..04c44850 100644
--- a/modules/sfp_tool_cmseek.py
+++ b/modules/sfp_tool_cmseek.py
@@ -15,7 +15,7 @@ from subprocess import Popen, PIPE
import io
import json
import os.path
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_tool_cmseek(SpiderFootPlugin):
"""Tool - CMSeeK:Footprint,Investigate:Content Analysis:tool:Identify what Content Management System (CMS) might be used."""
@@ -23,9 +23,9 @@ class sfp_tool_cmseek(SpiderFootPlugin):
meta = {
'name': "Tool - CMSeeK",
'summary': "Identify what Content Management System (CMS) might be used.",
- 'flags': [ "tool" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Content Analysis" ],
+ 'flags': ["tool"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Content Analysis"],
'toolDetails': {
'name': "CMSeeK",
'description': "CMSeek is a tool that is used to extract Content Management System(CMS) details of a website.",
diff --git a/modules/sfp_tool_dnstwist.py b/modules/sfp_tool_dnstwist.py
index 706c416f..4aa13a75 100644
--- a/modules/sfp_tool_dnstwist.py
+++ b/modules/sfp_tool_dnstwist.py
@@ -14,7 +14,7 @@
from subprocess import Popen, PIPE
import json
import os.path
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_tool_dnstwist(SpiderFootPlugin):
"""Tool - DNSTwist:Footprint,Investigate:DNS:tool:Identify bit-squatting, typo and other similar domains to the target using a local DNSTwist installation."""
@@ -22,9 +22,9 @@ class sfp_tool_dnstwist(SpiderFootPlugin):
meta = {
'name': "Tool - DNSTwist",
'summary': "Identify bit-squatting, typo and other similar domains to the target using a local DNSTwist installation.",
- 'flags': [ "tool" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "DNS" ],
+ 'flags': ["tool"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["DNS"],
'toolDetails': {
'name': "DNSTwist",
'description': "See what sort of trouble users can get in trying to type your domain name. "
diff --git a/modules/sfp_tool_nmap.py b/modules/sfp_tool_nmap.py
index c72d2514..9d562bbc 100644
--- a/modules/sfp_tool_nmap.py
+++ b/modules/sfp_tool_nmap.py
@@ -13,7 +13,7 @@
from subprocess import Popen, PIPE
import os.path
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_tool_nmap(SpiderFootPlugin):
"""Tool - Nmap:Footprint,Investigate:Crawling and Scanning:tool,slow,invasive:Identify what Operating System might be used."""
@@ -21,9 +21,9 @@ class sfp_tool_nmap(SpiderFootPlugin):
meta = {
'name': "Tool - Nmap",
'summary': "Identify what Operating System might be used.",
- 'flags': [ "tool", "slow", "invasive" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Crawling and Scanning" ],
+ 'flags': ["tool", "slow", "invasive"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Crawling and Scanning"],
'toolDetails': {
'name': "Nmap",
'description': "Nmap (\"Network Mapper\") is a free and open source utility for network discovery and security auditing.\n"
@@ -165,8 +165,7 @@ class sfp_tool_nmap(SpiderFootPlugin):
if "OS details:" in line:
junk, opsys = line.split(": ")
if opsys:
- evt = SpiderFootEvent("OPERATING_SYSTEM", opsys,
- self.__name__, event)
+ evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, event)
self.notifyListeners(evt)
except BaseException as e:
self.sf.error("Couldn't parse the output of Nmap: " + str(e), False)
@@ -183,16 +182,14 @@ class sfp_tool_nmap(SpiderFootPlugin):
junk, opsys = line.split(": ")
if opsys and currentIp:
- ipevent = SpiderFootEvent("IP_ADDRESS", currentIp,
- self.__name__, event)
+ ipevent = SpiderFootEvent("IP_ADDRESS", currentIp, self.__name__, event)
self.notifyListeners(ipevent)
- evt = SpiderFootEvent("OPERATING_SYSTEM", opsys,
- self.__name__, ipevent)
+ evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, ipevent)
self.notifyListeners(evt)
currentIp = None
except BaseException as e:
- self.sf.error("Couldn't parse the output of Nmap: " + str(e), False)
+ self.sf.error(f"Couldn't parse the output of Nmap: {e}", False)
return None
# End of sfp_tool_nmap class
diff --git a/modules/sfp_tool_whatweb.py b/modules/sfp_tool_whatweb.py
index 88dafa7c..2c17e5a9 100644
--- a/modules/sfp_tool_whatweb.py
+++ b/modules/sfp_tool_whatweb.py
@@ -14,7 +14,7 @@
from subprocess import Popen, PIPE
import json
import os.path
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_tool_whatweb(SpiderFootPlugin):
"""Tool - WhatWeb:Footprint,Investigate:Content Analysis:tool:Identify what software is in use on the specified website."""
@@ -22,9 +22,9 @@ class sfp_tool_whatweb(SpiderFootPlugin):
meta = {
'name': "Tool - WhatWeb",
'summary': "Identify what software is in use on the specified website.",
- 'flags': [ "tool" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Content Analysis" ],
+ 'flags': ["tool"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Content Analysis"],
'toolDetails': {
'name': "WhatWeb",
'description': "WhatWeb identifies websites. Its goal is to answer the question, \"What is that Website?\". "
diff --git a/modules/sfp_torch.py b/modules/sfp_torch.py
index f23133cc..fe067575 100644
--- a/modules/sfp_torch.py
+++ b/modules/sfp_torch.py
@@ -11,7 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import re
@@ -21,9 +21,9 @@ class sfp_torch(SpiderFootPlugin):
meta = {
'name': "TORCH",
'summary': "Search Tor 'TORCH' search engine for mentions of the target domain.",
- 'flags': [ "errorprone" ],
- 'useCases': [ "Footprint", "Investigate" ],
- 'categories': [ "Search Engines" ]
+ 'flags': ["errorprone"],
+ 'useCases': ["Footprint", "Investigate"],
+ 'categories': ["Search Engines"]
}
# Default options
@@ -62,7 +62,6 @@ class sfp_torch(SpiderFootPlugin):
def handleEvent(self, event):
eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':
@@ -71,24 +70,26 @@ class sfp_torch(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData + ", skipping.")
return None
- else:
- self.results[eventData] = True
- formpage = self.sf.fetchUrl("http://xmh57jrzrnw6insl.onion",
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout'])
+ self.results[eventData] = True
+
+ formpage = self.sf.fetchUrl(
+ "http://xmh57jrzrnw6insl.onion",
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']
+ )
if not formpage['content']:
self.sf.info("Couldn't connect to TORCH, check that you have TOR enabled.")
return None
- else:
- # Need the form ID to submit later for the search
- m = re.findall("\<form method=\"get\" action=\"/(\S+)/search.cgi\"\>",
- formpage['content'], re.IGNORECASE | re.DOTALL)
- if not m:
- return None
- formid = m[0]
+ # Need the form ID to submit later for the search
+ m = re.findall(r"\<form method=\"get\" action=\"/(\S+)/search.cgi\"\>",
+ formpage['content'], re.IGNORECASE | re.DOTALL)
+ if not m:
+ return None
+
+ formid = m[0]
pagecontent = ""
pagecount = 0
@@ -119,7 +120,7 @@ class sfp_torch(SpiderFootPlugin):
self.__name__, event)
self.notifyListeners(evt)
- links = re.findall("\<DT\>\d+.\s+<a href=\"(.*?)\"\s+TARGET=\"_blank\"\>",
+ links = re.findall(r"\<DT\>\d+.\s+<a href=\"(.*?)\"\s+TARGET=\"_blank\"\>",
data['content'], re.IGNORECASE | re.DOTALL)
for link in links:
@@ -149,7 +150,7 @@ class sfp_torch(SpiderFootPlugin):
try:
startIndex = res['content'].index(eventData) - 120
endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
+ except BaseException:
self.sf.debug("String not found in content.")
continue
diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.py
index 709544d2..78d3a3ff 100644
--- a/modules/sfp_torexits.py
+++ b/modules/sfp_torexits.py
@@ -13,14 +13,14 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'TOR Exist List': {
'id': '_torexits',
'checks': ['ip', 'netblock'],
'url': 'https://check.torproject.org/exit-addresses',
- 'regex': '^ExitAddress\s+{0}\s+'
+ 'regex': r'^ExitAddress\s+{0}\s+'
}
}
@@ -31,9 +31,9 @@ class sfp_torexits(SpiderFootPlugin):
meta = {
'name': "TOR Exit Nodes",
'summary': "Check if an IP or netblock appears on the torproject.org exit node list.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Secondary Networks" ]
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Secondary Networks"]
}
# Default options
@@ -111,8 +111,7 @@ class sfp_torexits(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_totalhash.py b/modules/sfp_totalhash.py
index 09eb279d..f29ae8d5 100644
--- a/modules/sfp_totalhash.py
+++ b/modules/sfp_totalhash.py
@@ -12,7 +12,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'TotalHash.com Database': {
@@ -31,9 +31,9 @@ class sfp_totalhash(SpiderFootPlugin):
meta = {
'name': "TotalHash.com",
'summary': "Check if a host/domain or IP is malicious according to TotalHash.com.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://totalhash.cymru.com/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_twilio.py b/modules/sfp_twilio.py
index 2243d360..fd530dcc 100644
--- a/modules/sfp_twilio.py
+++ b/modules/sfp_twilio.py
@@ -10,7 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
import base64
import json
@@ -20,9 +20,9 @@ class sfp_twilio(SpiderFootPlugin):
meta = {
'name': "Twilio",
'summary': "Obtain information from Twilio about phone numbers. Ensure you have the Caller Name add-on installed in Twilio.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.twilio.com/",
'model': "FREE_AUTH_LIMITED",
diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.py
index a5de6135..9d5b6a23 100644
--- a/modules/sfp_twitter.py
+++ b/modules/sfp_twitter.py
@@ -11,7 +11,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_twitter(SpiderFootPlugin):
"""Twitter:Footprint,Investigate,Passive:Social Media::Gather name and location from Twitter profiles."""
@@ -19,9 +19,9 @@ class sfp_twitter(SpiderFootPlugin):
meta = {
'name': "Twitter",
'summary': "Gather name and location from Twitter profiles.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://twitter.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -52,11 +52,11 @@ class sfp_twitter(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ "SOCIAL_MEDIA" ]
+ return ["SOCIAL_MEDIA"]
# What events this module produces
def producedEvents(self):
- return [ "RAW_RIR_DATA", "GEOINFO" ]
+ return ["RAW_RIR_DATA", "GEOINFO"]
# Handle events sent to this module
def handleEvent(self, event):
diff --git a/modules/sfp_uceprotect.py b/modules/sfp_uceprotect.py
index baab3c19..cd17fcc0 100644
--- a/modules/sfp_uceprotect.py
+++ b/modules/sfp_uceprotect.py
@@ -13,7 +13,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_uceprotect(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_uceprotect(SpiderFootPlugin):
meta = {
'name': "UCEPROTECT",
'summary': "Query the UCEPROTECT databases for open relays, open proxies, vulnerable servers, etc.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.uceprotect.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -144,7 +144,6 @@ class sfp_uceprotect(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
parentEvent = event
- addrlist = list()
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
diff --git a/modules/sfp_urlscan.py b/modules/sfp_urlscan.py
index 5be82df2..7f28472a 100644
--- a/modules/sfp_urlscan.py
+++ b/modules/sfp_urlscan.py
@@ -12,7 +12,7 @@
import json
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_urlscan(SpiderFootPlugin):
"""URLScan.io:Footprint,Investigate,Passive:Search Engines::Search URLScan.io cache for domain information."""
@@ -20,9 +20,9 @@ class sfp_urlscan(SpiderFootPlugin):
meta = {
'name': "URLScan.io",
'summary': "Search URLScan.io cache for domain information.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://urlscan.io/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -91,7 +91,7 @@ class sfp_urlscan(SpiderFootPlugin):
try:
result = json.loads(res['content'])
except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ self.sf.debug(f"Error processing JSON response: {e}")
return None
return result
diff --git a/modules/sfp_venmo.py b/modules/sfp_venmo.py
index 6269bc95..8cae6cc1 100644
--- a/modules/sfp_venmo.py
+++ b/modules/sfp_venmo.py
@@ -11,7 +11,7 @@
import json
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_venmo(SpiderFootPlugin):
"""Venmo:Footprint,Investigate,Passive:Social Media::Gather user information from Venmo API."""
@@ -19,9 +19,9 @@ class sfp_venmo(SpiderFootPlugin):
meta = {
'name': "Venmo",
'summary': "Gather user information from Venmo API.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Social Media" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Social Media"],
'dataSource': {
'website': "https://venmo.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -51,11 +51,11 @@ class sfp_venmo(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return [ 'USERNAME' ]
+ return ['USERNAME']
# What events this module produces
def producedEvents(self):
- return [ 'RAW_RIR_DATA' ]
+ return ['RAW_RIR_DATA']
# Query Venmo API
def query(self, qry):
diff --git a/modules/sfp_viewdns.py b/modules/sfp_viewdns.py
index 25ce5718..30fad380 100644
--- a/modules/sfp_viewdns.py
+++ b/modules/sfp_viewdns.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_viewdns(SpiderFootPlugin):
"""ViewDNS.info:Investigate,Passive:Search Engines:apikey:Reverse Whois lookups using ViewDNS.info."""
@@ -19,9 +19,9 @@ class sfp_viewdns(SpiderFootPlugin):
meta = {
'name': "ViewDNS.info",
'summary': "Reverse Whois lookups using ViewDNS.info.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://viewdns.info/",
'model': "FREE_AUTH_LIMITED",
@@ -108,7 +108,7 @@ class sfp_viewdns(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot")
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("ViewDNS.info API key seems to have been rejected or you have exceeded usage limits.", False)
self.errorState = True
return None
@@ -196,7 +196,7 @@ class sfp_viewdns(SpiderFootPlugin):
myres.append(h.lower())
else:
continue
- if h.lower() in [ "demo1.com", "demo2.com", "demo3.com", "demo4.com", "demo5.com" ]:
+ if h.lower() in ["demo1.com", "demo2.com", "demo3.com", "demo4.com", "demo5.com"]:
continue
if eventName == "EMAILADDR":
e = SpiderFootEvent("AFFILIATE_INTERNET_NAME", h, self.__name__, event)
diff --git a/modules/sfp_virustotal.py b/modules/sfp_virustotal.py
index 3fca9192..d5374486 100644
--- a/modules/sfp_virustotal.py
+++ b/modules/sfp_virustotal.py
@@ -13,7 +13,7 @@
import json
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_virustotal(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_virustotal(SpiderFootPlugin):
meta = {
'name': "VirusTotal",
'summary': "Obtain information from VirusTotal about identified IP addresses.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://www.virustotal.com/",
'model': "FREE_AUTH_LIMITED",
@@ -123,7 +123,7 @@ class sfp_virustotal(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from VirusTotal.", False)
+ self.sf.error(f"Error processing JSON response from VirusTotal: {e}", False)
self.errorState = True
return None
@@ -226,7 +226,7 @@ class sfp_virustotal(SpiderFootPlugin):
# Treat siblings as affiliates if they are of the original target, otherwise
# they are additional hosts within the target.
if 'domain_siblings' in info:
- if eventName in [ "IP_ADDRESS", "INTERNET_NAME"]:
+ if eventName in ["IP_ADDRESS", "INTERNET_NAME"]:
for s in info['domain_siblings']:
if self.getTarget().matches(s):
if s not in self.results:
diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.py
index 7e06350f..df9368a6 100644
--- a/modules/sfp_voipbl.py
+++ b/modules/sfp_voipbl.py
@@ -13,14 +13,14 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'VOIPBL Publicly Accessible PBX List': {
'id': '_voipbl',
'checks': ['ip', 'netblock'],
'url': 'http://www.voipbl.org/update',
- 'regex': '{0}\/'
+ 'regex': r'{0}\/'
}
}
@@ -31,9 +31,9 @@ class sfp_voipbl(SpiderFootPlugin):
meta = {
'name': "VoIPBL OpenPBX IPs",
'summary': "Check if an IP or netblock is an open PBX according to VoIPBL OpenPBX IPs.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://www.voipbl.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -123,8 +123,7 @@ class sfp_voipbl(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.py
index 5df2a70d..4ffd78d4 100644
--- a/modules/sfp_vxvault.py
+++ b/modules/sfp_vxvault.py
@@ -13,14 +13,14 @@
from netaddr import IPAddress, IPNetwork
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'VXVault Malicious URL List': {
'id': '_vxvault',
'checks': ['ip', 'domain'],
'url': 'http://vxvault.net/URL_List.php',
- 'regex': '.*\/{0}/.*'
+ 'regex': r'.*\/{0}/.*'
}
}
@@ -31,9 +31,9 @@ class sfp_vxvault(SpiderFootPlugin):
meta = {
'name': "VXVault.net",
'summary': "Check if a domain or IP is malicious according to VXVault.net.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://vxvault.net/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -124,8 +124,7 @@ class sfp_vxvault(SpiderFootPlugin):
# build a list of IP.
# Cycle through each IP and check if it's in the netblock.
if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
pat = re.compile(rx, re.IGNORECASE)
self.sf.debug("New regex for " + check + ": " + rx)
for line in data['content'].split('\n'):
diff --git a/modules/sfp_watchguard.py b/modules/sfp_watchguard.py
index 1597bccf..14d0f6f3 100644
--- a/modules/sfp_watchguard.py
+++ b/modules/sfp_watchguard.py
@@ -12,12 +12,12 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
malchecks = {
'Watchguard Reputation Authority Lookup': {
'id': '_watchguard',
- 'checks': [ 'ip' ],
+ 'checks': ['ip'],
'url': 'http://reputationauthority.org/lookup?ip={0}',
'badregex': ['.*>[6-9][0-9]/100 </td>.*', '.*>100/100 </td>.*'],
'goodregex': []
@@ -31,9 +31,9 @@ class sfp_watchguard(SpiderFootPlugin):
meta = {
'name': "Watchguard",
'summary': "Check if an IP is malicious according to Watchguard's reputationauthority.org.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "http://reputationauthority.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -82,7 +82,7 @@ class sfp_watchguard(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR" ]
+ return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR"]
# Check the regexps to see whether the content indicates maliciousness
def contentMalicious(self, content, goodregex, badregex):
diff --git a/modules/sfp_webanalytics.py b/modules/sfp_webanalytics.py
index 819da742..45163cda 100644
--- a/modules/sfp_webanalytics.py
+++ b/modules/sfp_webanalytics.py
@@ -13,7 +13,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_webanalytics(SpiderFootPlugin):
"""Web Analytics Extractor:Footprint,Investigate,Passive:Content Analysis::Identify web analytics IDs in scraped webpages and DNS TXT records."""
@@ -21,9 +21,9 @@ class sfp_webanalytics(SpiderFootPlugin):
meta = {
'name': "Web Analytics Extractor",
'summary': "Identify web analytics IDs in scraped webpages and DNS TXT records.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
opts = {}
diff --git a/modules/sfp_webframework.py b/modules/sfp_webframework.py
index 09361539..d9bfcef6 100644
--- a/modules/sfp_webframework.py
+++ b/modules/sfp_webframework.py
@@ -12,18 +12,18 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
regexps = dict({
"jQuery": list(['jquery']), # unlikely false positive
- "YUI": list(['\/yui\/', 'yui\-', 'yui\.']),
- "Prototype": list(['\/prototype\/', 'prototype\-', 'prototype\.js']),
- "ZURB Foundation": list(['\/foundation\/', 'foundation\-', 'foundation\.js']),
- "Bootstrap": list(['\/bootstrap\/', 'bootstrap\-', 'bootstrap\.js']),
- "ExtJS": list(['[\'\"\=]ext\.js', 'extjs', '\/ext\/*\.js']),
- "Mootools": list(['\/mootools\/', 'mootools\-', 'mootools\.js']),
- "Dojo": list(['\/dojo\/', '[\'\"\=]dojo\-', '[\'\"\=]dojo\.js']),
- "Wordpress": list(['\/wp-includes\/', '\/wp-content\/'])
+ "YUI": list([r'\/yui\/', r'yui\-', r'yui\.']),
+ "Prototype": list([r'\/prototype\/', r'prototype\-', r'prototype\.js']),
+ "ZURB Foundation": list([r'\/foundation\/', r'foundation\-', r'foundation\.js']),
+ "Bootstrap": list([r'\/bootstrap\/', r'bootstrap\-', r'bootstrap\.js']),
+ "ExtJS": list([r'[\'\"\=]ext\.js', 'extjs', r'\/ext\/*\.js']),
+ "Mootools": list([r'\/mootools\/', r'mootools\-', r'mootools\.js']),
+ "Dojo": list([r'\/dojo\/', r'[\'\"\=]dojo\-', r'[\'\"\=]dojo\.js']),
+ "Wordpress": list([r'\/wp-includes\/', r'\/wp-content\/'])
})
class sfp_webframework(SpiderFootPlugin):
@@ -32,9 +32,9 @@ class sfp_webframework(SpiderFootPlugin):
meta = {
'name': "Web Framework Identifier",
'summary': "Identify the usage of popular web frameworks like jQuery, YUI and others.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.py
index 436c2a16..246af5da 100644
--- a/modules/sfp_webserver.py
+++ b/modules/sfp_webserver.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_webserver(SpiderFootPlugin):
"""Web Server Identifier:Footprint,Investigate,Passive:Content Analysis::Obtain web server banners to identify versions of web servers being used."""
@@ -20,9 +20,9 @@ class sfp_webserver(SpiderFootPlugin):
meta = {
'name': "Web Server Identifier",
'summary': "Obtain web server banners to identify versions of web servers being used.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
@@ -71,7 +71,7 @@ class sfp_webserver(SpiderFootPlugin):
jdata = json.loads(eventData)
if jdata == None:
return None
- except BaseException as e:
+ except BaseException:
self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
return None
@@ -115,7 +115,7 @@ class sfp_webserver(SpiderFootPlugin):
return None
tech = None
- if 'set-cookie'in jdata and 'PHPSESS' in jdata['set-cookie']:
+ if 'set-cookie' in jdata and 'PHPSESS' in jdata['set-cookie']:
tech = "PHP"
if 'set-cookie' in jdata and 'JSESSIONID' in jdata['set-cookie']:
diff --git a/modules/sfp_whatcms.py b/modules/sfp_whatcms.py
index 52b292ad..e8b29d5e 100644
--- a/modules/sfp_whatcms.py
+++ b/modules/sfp_whatcms.py
@@ -13,7 +13,7 @@
import json
import urllib.request, urllib.parse, urllib.error
import time
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_whatcms(SpiderFootPlugin):
"""WhatCMS:Footprint,Investigate,Passive:Content Analysis:apikey,slow:Check web technology using WhatCMS.org API."""
@@ -21,9 +21,9 @@ class sfp_whatcms(SpiderFootPlugin):
meta = {
'name': "WhatCMS",
'summary': "Check web technology using WhatCMS.org API.",
- 'flags': [ "apikey", "slow" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Content Analysis" ]
+ 'flags': ["apikey", "slow"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Content Analysis"]
}
# Default options
diff --git a/modules/sfp_whois.py b/modules/sfp_whois.py
index c6c4ea89..2d7dcbd2 100644
--- a/modules/sfp_whois.py
+++ b/modules/sfp_whois.py
@@ -14,7 +14,7 @@
import whois
import ipwhois
from netaddr import IPAddress
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_whois(SpiderFootPlugin):
"""Whois:Footprint,Investigate,Passive:Public Registries::Perform a WHOIS look-up on domain names and owned netblocks."""
@@ -22,9 +22,9 @@ class sfp_whois(SpiderFootPlugin):
meta = {
'name': "Whois",
'summary': "Perform a WHOIS look-up on domain names and owned netblocks.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Public Registries" ]
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Public Registries"]
}
# Default options
@@ -47,7 +47,7 @@ class sfp_whois(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
return ["DOMAIN_NAME", "DOMAIN_NAME_PARENT", "NETBLOCK_OWNER",
- "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN_NAME", "SIMILARDOMAIN" ]
+ "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN_NAME", "SIMILARDOMAIN"]
# What events this module produces
# This is to support the end user in selecting modules based on events
diff --git a/modules/sfp_whoisology.py b/modules/sfp_whoisology.py
index 9d321681..b6e59d1e 100644
--- a/modules/sfp_whoisology.py
+++ b/modules/sfp_whoisology.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_whoisology(SpiderFootPlugin):
"""Whoisology:Investigate,Passive:Search Engines:apikey:Reverse Whois lookups using Whoisology.com."""
@@ -19,9 +19,9 @@ class sfp_whoisology(SpiderFootPlugin):
meta = {
'name': "Whoisology",
'summary': "Reverse Whois lookups using Whoisology.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://whoisology.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -81,7 +81,7 @@ class sfp_whoisology(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot")
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("Whoisology API key seems to have been rejected or you have exceeded usage limits.", False)
self.errorState = True
return None
diff --git a/modules/sfp_whoxy.py b/modules/sfp_whoxy.py
index f2903917..7ded0fee 100644
--- a/modules/sfp_whoxy.py
+++ b/modules/sfp_whoxy.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import json
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_whoxy(SpiderFootPlugin):
"""Whoxy:Investigate,Passive:Search Engines:apikey:Reverse Whois lookups using Whoxy.com."""
@@ -19,9 +19,9 @@ class sfp_whoxy(SpiderFootPlugin):
meta = {
'name': "Whoxy",
'summary': "Reverse Whois lookups using Whoxy.com.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Search Engines" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Search Engines"],
'dataSource': {
'website': "https://www.whoxy.com/",
'model': "COMMERCIAL_ONLY",
@@ -89,7 +89,7 @@ class sfp_whoxy(SpiderFootPlugin):
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot")
- if res['code'] in [ "400", "429", "500", "403" ]:
+ if res['code'] in ["400", "429", "500", "403"]:
self.sf.error("Whoxy API key seems to have been rejected or you have exceeded usage limits.", False)
self.errorState = True
return None
diff --git a/modules/sfp_wigle.py b/modules/sfp_wigle.py
index 8ec5c519..c70aa797 100644
--- a/modules/sfp_wigle.py
+++ b/modules/sfp_wigle.py
@@ -12,17 +12,17 @@
import json
import datetime
import urllib.request, urllib.parse, urllib.error
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_wigle(SpiderFootPlugin):
- """Wigle.net:Footprint,Investigate,Passive:Secondary Networks:apikey:Query wigle.net to identify nearby WiFi access points."""
+ """WiGLE:Footprint,Investigate,Passive:Secondary Networks:apikey:Query WiGLE to identify nearby WiFi access points."""
meta = {
- 'name': "Wigle.net",
- 'summary': "Query wigle.net to identify nearby WiFi access points.",
- 'flags': [ "apikey" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Secondary Networks" ],
+ 'name': "WiGLE",
+ 'summary': "Query WiGLE to identify nearby WiFi access points.",
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Secondary Networks"],
'dataSource': {
'website': "https://wigle.net/",
'model': "FREE_AUTH_UNLIMITED",
@@ -107,7 +107,7 @@ class sfp_wigle(SpiderFootPlugin):
return None
return info['results'][0]['boundingbox']
except Exception as e:
- self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from Wigle.net: {e}", False)
return None
def getnetworks(self, coords):
@@ -144,7 +144,7 @@ class sfp_wigle(SpiderFootPlugin):
ret.append(r['ssid'] + " (Net ID: " + r['netid'] + ")")
return ret
except Exception as e:
- self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)
+ self.sf.error(f"Error processing JSON response from WiGLE: {e}", False)
return None
diff --git a/modules/sfp_wikileaks.py b/modules/sfp_wikileaks.py
index 9e821899..83b00fb4 100644
--- a/modules/sfp_wikileaks.py
+++ b/modules/sfp_wikileaks.py
@@ -11,7 +11,7 @@
# -------------------------------------------------------------------------------
import datetime
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_wikileaks(SpiderFootPlugin):
@@ -20,9 +20,9 @@ class sfp_wikileaks(SpiderFootPlugin):
meta = {
'name': "Wikileaks",
'summary': "Search Wikileaks for mentions of domain names and e-mail addresses.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://wikileaks.org/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_wikipediaedits.py b/modules/sfp_wikipediaedits.py
index e287c217..d4ad9d73 100644
--- a/modules/sfp_wikipediaedits.py
+++ b/modules/sfp_wikipediaedits.py
@@ -14,7 +14,7 @@ import datetime
import re
import urllib.request, urllib.parse, urllib.error
from html.parser import HTMLParser
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_wikipediaedits(SpiderFootPlugin):
"""Wikipedia Edits:Footprint,Investigate,Passive:Secondary Networks::Identify edits to Wikipedia articles made from a given IP address or username."""
@@ -22,9 +22,9 @@ class sfp_wikipediaedits(SpiderFootPlugin):
meta = {
'name': "Wikipedia Edits",
'summary': "Identify edits to Wikipedia articles made from a given IP address or username.",
- 'flags': [ "" ],
- 'useCases': [ "Footprint", "Investigate", "Passive" ],
- 'categories': [ "Secondary Networks" ],
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Secondary Networks"],
'dataSource': {
'website': "https://www.wikipedia.org/",
'model': "FREE_NOAUTH_UNLIMITED",
diff --git a/modules/sfp_xforce.py b/modules/sfp_xforce.py
index 37d1a71d..00cd5711 100644
--- a/modules/sfp_xforce.py
+++ b/modules/sfp_xforce.py
@@ -16,7 +16,7 @@ import base64
from datetime import datetime
import time
from netaddr import IPNetwork
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_xforce(SpiderFootPlugin):
"""XForce Exchange:Investigate,Passive:Reputation Systems:apikey:Obtain information from IBM X-Force Exchange"""
@@ -24,9 +24,9 @@ class sfp_xforce(SpiderFootPlugin):
meta = {
'name': "XForce Exchange",
'summary': "Obtain information from IBM X-Force Exchange",
- 'flags': [ "apikey" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': ["apikey"],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://exchange.xforce.ibmcloud.com/",
'model': "FREE_AUTH_LIMITED",
@@ -110,8 +110,6 @@ class sfp_xforce(SpiderFootPlugin):
"CO_HOSTED_SITE"]
def query(self, qry, querytype):
- ret = None
-
if querytype not in ["ipr/malware", "ipr/history", "resolve"]:
querytype = "ipr/malware"
@@ -176,7 +174,7 @@ class sfp_xforce(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- self.sf.error("Error processing JSON response from X-Force Exchange.", False)
+ self.sf.error(f"Error processing JSON response from X-Force Exchange: {e}", False)
return None
return info
@@ -287,7 +285,6 @@ class sfp_xforce(SpiderFootPlugin):
if len(rec_history) > 0:
self.sf.debug("Found history results in XForce")
for result in rec_history:
- reasonDescription = result.get("reasonDescription", "")
created = result.get("created", None)
# 2014-11-06T10:45:00.000Z
if not created:
@@ -316,7 +313,7 @@ class sfp_xforce(SpiderFootPlugin):
self.notifyListeners(e)
# ipr/malware doesn't support hostnames
- if eventName in [ "CO_HOSTED_SITE", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME" ]:
+ if eventName in ["CO_HOSTED_SITE", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME"]:
continue
rec = self.query(addr, "ipr/malware")
@@ -325,7 +322,6 @@ class sfp_xforce(SpiderFootPlugin):
if len(rec_malware) > 0:
self.sf.debug("Found malware results in XForce")
for result in rec_malware:
- count = result.get("count", "")
origin = result.get("origin", "")
domain = result.get("domain", "")
uri = result.get("uri", "")
diff --git a/modules/sfp_yandexdns.py b/modules/sfp_yandexdns.py
index 9dabc570..1aeff2d3 100644
--- a/modules/sfp_yandexdns.py
+++ b/modules/sfp_yandexdns.py
@@ -12,7 +12,7 @@
# -------------------------------------------------------------------------------
import dns.resolver
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_yandexdns(SpiderFootPlugin):
@@ -21,9 +21,9 @@ class sfp_yandexdns(SpiderFootPlugin):
meta = {
'name': "Yandex DNS",
'summary': "Check if a host would be blocked by Yandex DNS",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Reputation Systems" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Reputation Systems"],
'dataSource': {
'website': "https://yandex.com/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -70,13 +70,13 @@ class sfp_yandexdns(SpiderFootPlugin):
# https://dns.yandex.com/advanced/
def queryAddr(self, qaddr):
res = dns.resolver.Resolver()
- res.nameservers = [ "77.88.8.88", "77.88.8.2" ]
+ res.nameservers = ["77.88.8.88", "77.88.8.2"]
try:
addrs = res.query(qaddr)
- self.sf.debug("Addresses returned: " + str(addrs))
- except BaseException as e:
- self.sf.debug("Unable to resolve " + qaddr)
+ self.sf.debug(f"Addresses returned: {addrs}")
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {qaddr}")
return False
if addrs:
@@ -103,8 +103,8 @@ class sfp_yandexdns(SpiderFootPlugin):
try:
if self.sf.resolveHost(eventData):
resolved = True
- except BaseException as e:
- self.sf.debug("Unable to resolve " + eventData)
+ except BaseException:
+ self.sf.debug(f"Unable to resolve {eventData}")
return None
if not resolved:
diff --git a/modules/sfp_zoneh.py b/modules/sfp_zoneh.py
index f0ce8f52..1733e7e1 100644
--- a/modules/sfp_zoneh.py
+++ b/modules/sfp_zoneh.py
@@ -13,7 +13,7 @@
import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+from sflib import SpiderFootPlugin, SpiderFootEvent
class sfp_zoneh(SpiderFootPlugin):
@@ -22,9 +22,9 @@ class sfp_zoneh(SpiderFootPlugin):
meta = {
'name': "Zone-H Defacement Check",
'summary': "Check if a hostname/domain appears on the zone-h.org 'special defacements' RSS feed.",
- 'flags': [ "" ],
- 'useCases': [ "Investigate", "Passive" ],
- 'categories': [ "Leaks, Dumps and Breaches" ],
+ 'flags': [""],
+ 'useCases': ["Investigate", "Passive"],
+ 'categories': ["Leaks, Dumps and Breaches"],
'dataSource': {
'website': "https://zone-h.org/",
'model': "FREE_NOAUTH_UNLIMITED",
@@ -88,7 +88,7 @@ class sfp_zoneh(SpiderFootPlugin):
"DEFACED_COHOST", "DEFACED_AFFILIATE_IPADDR"]
def lookupItem(self, target, content):
- grps = re.findall("<title><\!\[CDATA\[(.[^\]]*)\]\]></title>\s+<link><\!\[CDATA\[(.[^\]]*)\]\]></link>", content)
+ grps = re.findall(r"<title><\!\[CDATA\[(.[^\]]*)\]\]></title>\s+<link><\!\[CDATA\[(.[^\]]*)\]\]></link>", content)
for m in grps:
if target in m[0]:
self.sf.info("Found zoneh site: " + m[0])
diff --git a/sf.py b/sf.py
index 12a46e08..62fa5c7f 100755
--- a/sf.py
+++ b/sf.py
@@ -139,7 +139,7 @@ if __name__ == '__main__':
(addr, port) = args.l.split(":")
sfConfig['__webaddr'] = addr
sfConfig['__webport'] = int(port)
- except BaseException as e:
+ except BaseException:
print("Invalid ip:port format.")
sys.exit(-1)
else:
diff --git a/sfdb.py b/sfdb.py
index 7e106459..16d376bc 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -294,7 +294,7 @@ class SpiderFootDb:
try:
rx = re.compile(qry, re.IGNORECASE | re.DOTALL)
ret = rx.match(data)
- except BaseException as e:
+ except BaseException:
return False
return ret is not None
@@ -317,7 +317,7 @@ class SpiderFootDb:
try:
self.dbh.execute(qry)
self.conn.commit()
- except BaseException as e:
+ except BaseException:
continue
self.conn.commit()
# self.conn.close()