SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
resolve flake8 violations
Brendan Colesa0d8e9eee084
diff --git a/modules/sfp__stor_db.py b/modules/sfp__stor_db.pyindex fd39cae3..65d59a1f 100644--- a/modules/sfp__stor_db.py+++ b/modules/sfp__stor_db.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin+from sflib import SpiderFootPluginclass sfp__stor_db(SpiderFootPlugin):diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.pyindex 423e872c..531240ad 100644--- a/modules/sfp__stor_stdout.py+++ b/modules/sfp__stor_stdout.py@@ -10,7 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin+from sflib import SpiderFootPluginimport json@@ -27,7 +27,7 @@ class sfp__stor_stdout(SpiderFootPlugin):# Default optionsopts = {- "_format": "tab", # tab, csv, json+ "_format": "tab", # tab, csv, json"_requested": [],"_showonlyrequested": False,"_stripnewline": False,diff --git a/modules/sfp_abusech.py b/modules/sfp_abusech.pyindex dbf07e88..65f30ce6 100644--- a/modules/sfp_abusech.py+++ b/modules/sfp_abusech.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'abuse.ch Zeus Tracker (Domain)': {@@ -58,8 +58,8 @@ class sfp_abusech(SpiderFootPlugin):'name': "abuse.ch",'summary': "Check if a host/domain, IP or netblock is malicious according to abuse.ch.",'flags': [],- 'useCases': [ "Passive", "Investigate" ],- 'categories': [ "Reputation Systems" ],+ 'useCases': ["Passive", "Investigate"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.abuse.ch",'model': "FREE_AUTH_UNLIMITED",@@ -184,8 +184,7 @@ class sfp_abusech(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_abuseipdb.py b/modules/sfp_abuseipdb.pyindex 7bc091d0..c17d0d39 100644--- a/modules/sfp_abuseipdb.py+++ b/modules/sfp_abuseipdb.py@@ -12,7 +12,7 @@import refrom netaddr import IPAddress, IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'AbuseIPDB': {@@ -28,9 +28,9 @@ class sfp_abuseipdb(SpiderFootPlugin):meta = {'name': "AbuseIPDB",'summary': "Check if an IP address is malicious according to AbuseIPDB.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Passive", "Investigate" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Passive", "Investigate"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.abuseipdb.com",'model': "FREE_AUTH_LIMITED",@@ -145,8 +145,7 @@ class sfp_abuseipdb(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex 22042eaa..54d51ceb 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -16,7 +16,7 @@ import threadingfrom queue import Queue, Empty as QueueEmptyimport jsonimport random-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_accounts(SpiderFootPlugin):"""Account Finder:Footprint,Passive:Social Media:slow:Look for possible associated accounts on nearly 200 websites like Ebay, Slashdot, reddit, etc."""@@ -24,8 +24,8 @@ class sfp_accounts(SpiderFootPlugin):meta = {'name': "Account Finder",'summary': "Look for possible associated accounts on nearly 200 websites like Ebay, Slashdot, reddit, etc.",- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Social Media" ]+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Social Media"]}# Default options@@ -85,7 +85,7 @@ class sfp_accounts(SpiderFootPlugin):try:self.sites = [site for site in json.loads(content)['sites'] if site['valid']]except BaseException as e:- self.sf.error("Unable to parse social media accounts list.", False)+ self.sf.error(f"Unable to parse social media accounts list: {e}", False)self.errorState = Truereturn None@@ -150,8 +150,8 @@ class sfp_accounts(SpiderFootPlugin):site = queue.get(timeout=0.1)try:self.checkSite(username, site)- except Exception as ex:- self.sf.debug(f'thread {threading.current_thread().name} exception {ex}')+ except Exception as e:+ self.sf.debug(f'thread {threading.current_thread().name} exception {e}')except QueueEmpty:return@@ -240,7 +240,7 @@ class sfp_accounts(SpiderFootPlugin):self.distrustedChecked = Trueif eventName == "HUMAN_NAME":- names = [ eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".") ]+ names = [eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".")]for name in names:users.append(name)@@ -259,7 +259,6 @@ class sfp_accounts(SpiderFootPlugin):users.append(eventData)for user in users:- adduser = Trueif user in self.opts['_genericusers'].split(","):self.sf.debug(user + " is a generic account name, skipping.")continuediff --git a/modules/sfp_adblock.py b/modules/sfp_adblock.pyindex 69c51030..3ddd36a9 100644--- a/modules/sfp_adblock.py+++ b/modules/sfp_adblock.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import adblockparser-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_adblock(SpiderFootPlugin):@@ -21,8 +21,8 @@ class sfp_adblock(SpiderFootPlugin):meta = {'name': "AdBlock Check",'summary': "Check if linked pages would be blocked by AdBlock Plus.",- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://adblockplus.org/",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_ahmia.py b/modules/sfp_ahmia.pyindex 11e3ab69..d88fffd7 100644--- a/modules/sfp_ahmia.py+++ b/modules/sfp_ahmia.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport re@@ -21,8 +21,8 @@ class sfp_ahmia(SpiderFootPlugin):meta = {'name': "Ahmia",'summary': "Search Tor 'Ahmia' search engine for mentions of the target domain.",- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Search Engines" ],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://ahmia.fi/",'model': "FREE_NOAUTH_UNLIMITED",@@ -80,7 +80,6 @@ class sfp_ahmia(SpiderFootPlugin):def handleEvent(self, event):eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif not self.opts['fullnames'] and eventName == 'HUMAN_NAME':@@ -105,8 +104,7 @@ class sfp_ahmia(SpiderFootPlugin):if self.checkForStop():return None- links = re.findall("redirect_url=(.[^\"]+)\"",- data['content'], re.IGNORECASE | re.DOTALL)+ links = re.findall("redirect_url=(.[^\"]+)\"", data['content'], re.IGNORECASE | re.DOTALL)reported = Falsefor link in links:@@ -137,7 +135,7 @@ class sfp_ahmia(SpiderFootPlugin):try:startIndex = res['content'].index(eventData) - 120endIndex = startIndex + len(eventData) + 240- except BaseException as e:+ except BaseException:self.sf.debug("String not found in content.")continuediff --git a/modules/sfp_alienvault.py b/modules/sfp_alienvault.pyindex cd7ad3ce..02965cf6 100644--- a/modules/sfp_alienvault.py+++ b/modules/sfp_alienvault.py@@ -14,7 +14,7 @@ import jsonfrom datetime import datetimeimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_alienvault(SpiderFootPlugin):"""AlienVault OTX:Investigate,Passive:Reputation Systems:apikey:Obtain information from AlienVault Open Threat Exchange (OTX)"""@@ -22,9 +22,9 @@ class sfp_alienvault(SpiderFootPlugin):meta = {'name': "AlienVault OTX",'summary': "Obtain information from AlienVault Open Threat Exchange (OTX)",- 'flags': [ "apikey"],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://otx.alienvault.com/",'model': "FREE_AUTH_LIMITED",@@ -105,10 +105,9 @@ class sfp_alienvault(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK" ]+ return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK"]def query(self, qry, querytype):- ret = Nonetargettype = "hostname"if ":" in qry:@@ -141,12 +140,11 @@ class sfp_alienvault(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from AlienVault OTX.", False)+ self.sf.error(f"Error processing JSON response from AlienVault OTX: {e}", False)return Nonereturn info-# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventType@@ -221,7 +219,7 @@ class sfp_alienvault(SpiderFootPlugin):if self.opts['age_limit_days'] > 0 and last_ts < age_limit_ts:self.sf.debug("Record found but too old, skipping.")continue- except BaseException as e:+ except BaseException:self.sf.debug("Couldn't parse date from AlienVault so assuming it's OK.")e = SpiderFootEvent(evtType, host, self.__name__, event)self.notifyListeners(e)@@ -258,7 +256,7 @@ class sfp_alienvault(SpiderFootPlugin):if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:self.sf.debug("Record found but too old, skipping.")continue- except BaseException as e:+ except BaseException:self.sf.debug("Couldn't parse date from AlienVault so assuming it's OK.")# For netblocks, we need to create the IP address event so thatdiff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.pyindex fd4a5a84..3b6f269e 100644--- a/modules/sfp_alienvaultiprep.py+++ b/modules/sfp_alienvaultiprep.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'AlienVault IP Reputation Database': {@@ -31,8 +31,8 @@ class sfp_alienvaultiprep(SpiderFootPlugin):meta = {'name': "AlienVault IP Reputation",'summary': "Check if an IP or netblock is malicious according to the AlienVault IP Reputation database.",- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://cybersecurity.att.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -126,8 +126,7 @@ class sfp_alienvaultiprep(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_api_recon_dev.py b/modules/sfp_api_recon_dev.pyindex 4c1e3286..a5bb7519 100644--- a/modules/sfp_api_recon_dev.py+++ b/modules/sfp_api_recon_dev.py@@ -13,7 +13,7 @@import jsonimport timeimport urllib-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_api_recon_dev(SpiderFootPlugin):"""api.recon.dev:Footprint,Investigate,Passive:Passive DNS::Search api.recon.dev for subdomains."""@@ -21,9 +21,9 @@ class sfp_api_recon_dev(SpiderFootPlugin):meta = {'name': "api.recon.dev",'summary': "Search api.recon.dev for subdomains.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"]}opts = {@@ -55,7 +55,7 @@ class sfp_api_recon_dev(SpiderFootPlugin):def queryDomain(self, qry):headers = {- "Accept" : "application/json"+ "Accept": "application/json"}params = {'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')@@ -90,7 +90,7 @@ class sfp_api_recon_dev(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return None# returns list of results; 'null' when no results; or dict when there's an errordiff --git a/modules/sfp_apility.py b/modules/sfp_apility.pyindex 6b054f2a..0ca3880e 100644--- a/modules/sfp_apility.py+++ b/modules/sfp_apility.py@@ -13,7 +13,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_apility(SpiderFootPlugin):"""Apility:Footprint,Investigate,Passive:Reputation Systems:apikey:Search Apility API for IP address and domain reputation."""@@ -21,9 +21,9 @@ class sfp_apility(SpiderFootPlugin):meta = {'name': "Apility",'summary': "Search Apility API for IP address and domain reputation.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ]+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Reputation Systems"]}# Default options@@ -151,7 +151,7 @@ class sfp_apility(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn datadiff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.pyindex af3f8263..0c28c08f 100644--- a/modules/sfp_archiveorg.py+++ b/modules/sfp_archiveorg.py@@ -13,7 +13,7 @@import datetimeimport json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_archiveorg(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_archiveorg(SpiderFootPlugin):meta = {'name': "Archive.org",'summary': "Identifies historic versions of interesting files/pages from the Wayback Machine.",- 'flags': [ "slow" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["slow"],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://archive.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -150,11 +150,11 @@ class sfp_archiveorg(SpiderFootPlugin):try:ret = json.loads(res['content'])except BaseException as e:+ self.sf.debug(f"Error processing JSON response: {e}")ret = Noneif ret == None:- self.sf.error("Unable to process empty response from archive.org: " + \- eventData, False)+ self.sf.error("Unable to process empty response from archive.org: {eventData}", False)continueif len(ret['archived_snapshots']) < 1:diff --git a/modules/sfp_arin.py b/modules/sfp_arin.pyindex d5d9256a..86fc2b5f 100644--- a/modules/sfp_arin.py+++ b/modules/sfp_arin.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_arin(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_arin(SpiderFootPlugin):meta = {'name': "ARIN",'summary': "Queries ARIN registry for contact information.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Public Registries" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Public Registries"],'dataSource': {'website': "https://www.arin.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -80,7 +80,7 @@ class sfp_arin(SpiderFootPlugin):if url in self.memCache:res = self.memCache[url]else:- head = { "Accept": "application/json" }+ head = {"Accept": "application/json"}res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'], headers=head)if res['content'] is not None:@@ -92,7 +92,6 @@ class sfp_arin(SpiderFootPlugin):# Owner information about an ASdef query(self, qtype, value):- ownerinfo = dict()url = "https://whois.arin.net/rest/"if qtype == "domain":diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.pyindex 10842b76..fdfa0fa5 100644--- a/modules/sfp_azureblobstorage.py+++ b/modules/sfp_azureblobstorage.py@@ -13,7 +13,7 @@import threadingimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_azureblobstorage(SpiderFootPlugin):"""Azure Blob Finder:Footprint,Passive:Crawling and Scanning::Search for potential Azure blobs associated with the target and attempt to list their contents."""@@ -21,9 +21,9 @@ class sfp_azureblobstorage(SpiderFootPlugin):meta = {'name': "Azure Blob Finder",'summary': "Search for potential Azure blobs associated with the target and attempt to list their contents.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Crawling and Scanning" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Crawling and Scanning"],'dataSource': {'website': "https://azure.microsoft.com/en-in/services/storage/blobs/",'model': "FREE_NOAUTH_UNLIMITED"@@ -72,7 +72,6 @@ class sfp_azureblobstorage(SpiderFootPlugin):self.s3results[url] = Truedef threadSites(self, siteList):- ret = list()self.s3results = dict()running = Truei = 0@@ -145,7 +144,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):self.notifyListeners(evt)return None- targets = [ eventData.replace('.', '') ]+ targets = [eventData.replace('.', '')]kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])if kw:targets.append(kw)diff --git a/modules/sfp_badipscom.py b/modules/sfp_badipscom.pyindex a71dc908..1e1d6b63 100644--- a/modules/sfp_badipscom.py+++ b/modules/sfp_badipscom.py@@ -10,7 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_badipscom(SpiderFootPlugin):"""badips.com:Investigate,Passive:Reputation Systems::Check if an IP address is malicious according to BadIPs.com."""@@ -18,9 +18,9 @@ class sfp_badipscom(SpiderFootPlugin):meta = {'name': "badips.com",'summary': "Check if an IP address is malicious according to BadIPs.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.badips.com/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_badpackets.py b/modules/sfp_badpackets.pyindex e40b66bf..b646e604 100644--- a/modules/sfp_badpackets.py+++ b/modules/sfp_badpackets.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventfrom netaddr import IPNetworkimport urllib.request, urllib.parse, urllib.errorimport json@@ -22,9 +22,9 @@ class sfp_badpackets(SpiderFootPlugin):meta = {'name': "Bad Packets",'summary': "Obtain information about any malicious activities involving IP addresses found",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://badpackets.net",'model': "COMMERCIAL_ONLY",@@ -140,7 +140,7 @@ class sfp_badpackets(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from Bad Packets.", False)+ self.sf.error(f"Error processing JSON response from Bad Packets: {e}", False)return Nonereturn data@@ -167,28 +167,24 @@ class sfp_badpackets(SpiderFootPlugin):if eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True++ self.results[eventData] = Trueif eventName == 'NETBLOCK_OWNER':if not self.opts['netblocklookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:- self.sf.debug("Network size bigger than permitted: " +- str(IPNetwork(eventData).prefixlen) + " > " +- str(self.opts['maxnetblock']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:+ self.sf.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {self.opts['maxnetblock']}")+ return Noneif eventName == 'NETBLOCK_MEMBER':if not self.opts['subnetlookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:- self.sf.debug("Network size bigger than permitted: " +- str(IPNetwork(eventData).prefixlen) + " > " +- str(self.opts['maxsubnet']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:+ self.sf.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {self.opts['maxsubnet']}")+ return Noneqrylist = list()if eventName.startswith("NETBLOCK_"):@@ -246,7 +242,7 @@ class sfp_badpackets(SpiderFootPlugin):continueif maliciousIP:- maliciousIPDesc = "Bad Packets [ " + str(maliciousIP) + " ]\n"+ maliciousIPDesc = "Bad Packets [" + str(maliciousIP) + "]\n"try:category = record.get('tags')[0].get('category')diff --git a/modules/sfp_bambenek.py b/modules/sfp_bambenek.pyindex e989025a..58197716 100644--- a/modules/sfp_bambenek.py+++ b/modules/sfp_bambenek.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'Bambenek C&C IP List': {@@ -36,9 +36,9 @@ class sfp_bambenek(SpiderFootPlugin):meta = {'name': "Bambenek C&C List",'summary': "Check if a host/domain or IP appears on Bambenek Consulting's C&C tracker lists.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.bambenekconsulting.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -140,8 +140,7 @@ class sfp_bambenek(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_bgpview.py b/modules/sfp_bgpview.pyindex 08691919..be8b5d1c 100644--- a/modules/sfp_bgpview.py+++ b/modules/sfp_bgpview.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_bgpview(SpiderFootPlugin):"""BGPView:Investigate,Footprint,Passive:Search Engines::Obtain network information from BGPView API."""@@ -20,9 +20,9 @@ class sfp_bgpview(SpiderFootPlugin):meta = {'name': "BGPView",'summary': "Obtain network information from BGPView API.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Footprint", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Footprint", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://bgpview.io/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.pyindex e3338543..fdd35deb 100644--- a/modules/sfp_binaryedge.py+++ b/modules/sfp_binaryedge.py@@ -13,7 +13,7 @@import jsonimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_binaryedge(SpiderFootPlugin):"""BinaryEdge:Footprint,Investigate,Passive:Search Engines:apikey:Obtain information from BinaryEdge.io's Internet scanning systems about breaches, vulerabilities, torrents and passive DNS."""@@ -21,9 +21,9 @@ class sfp_binaryedge(SpiderFootPlugin):meta = {'name': "BinaryEdge",'summary': "Obtain information from BinaryEdge.io's Internet scanning systems about breaches, vulerabilities, torrents and passive DNS.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.binaryedge.io/",'model': "FREE_AUTH_LIMITED",@@ -104,7 +104,7 @@ class sfp_binaryedge(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):return ["IP_ADDRESS", "DOMAIN_NAME", "EMAILADDR",- "NETBLOCK_OWNER", "NETBLOCK_MEMBER" ]+ "NETBLOCK_OWNER", "NETBLOCK_MEMBER"]# What events this module producesdef producedEvents(self):@@ -114,7 +114,6 @@ class sfp_binaryedge(SpiderFootPlugin):"CO_HOSTED_SITE", "MALICIOUS_IPADDR"]def query(self, qry, querytype, page=1):- ret = Noneretarr = list()if self.errorState:@@ -141,7 +140,7 @@ class sfp_binaryedge(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot", headers=headers)- if res['code'] in [ "429", "500" ]:+ if res['code'] in ["429", "500"]:self.sf.error("BinaryEdge.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)self.errorState = Truereturn None@@ -157,7 +156,7 @@ class sfp_binaryedge(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from BinaryEdge.io.", False)+ self.sf.error(f"Error processing JSON response from BinaryEdge.io: {e}", False)return Noneif info.get('page') and info['total'] > info.get('pagesize', 100) * info.get('page', 0):@@ -407,7 +406,7 @@ class sfp_binaryedge(SpiderFootPlugin):banner = prec['result']['data']['service']['banner']e = SpiderFootEvent(evtbtype, banner, self.__name__, ev)self.notifyListeners(e)- except BaseException as e:+ except BaseException:self.sf.debug("No banner information found.")for addr in qrylist:diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.pyindex ec086b7b..3ea310e2 100644--- a/modules/sfp_bingsearch.py+++ b/modules/sfp_bingsearch.py@@ -9,7 +9,7 @@# Copyright: (c) Steve Micallef 2013# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_bingsearch(SpiderFootPlugin):@@ -18,9 +18,9 @@ class sfp_bingsearch(SpiderFootPlugin):meta = {'name': "Bing",'summary': "Obtain information from bing to identify sub-domains and links.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.bing.com/",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.pyindex 7608b15e..077cacee 100644--- a/modules/sfp_bingsharedip.py+++ b/modules/sfp_bingsharedip.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_bingsharedip(SpiderFootPlugin):@@ -20,9 +20,9 @@ class sfp_bingsharedip(SpiderFootPlugin):meta = {'name': "Bing (Shared IPs)",'summary': "Search Bing for hosts sharing the same IP.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.bing.com/",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_binstring.py b/modules/sfp_binstring.pyindex 2d5587a4..10d50a78 100644--- a/modules/sfp_binstring.py+++ b/modules/sfp_binstring.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import string-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_binstring(SpiderFootPlugin):"""Binary String Extractor:Footprint:Content Analysis:errorprone:Attempt to identify strings in binary content."""@@ -19,9 +19,9 @@ class sfp_binstring(SpiderFootPlugin):meta = {'name': "Binary String Extractor",'summary': "Attempt to identify strings in binary content.",- 'flags': [ "errorprone" ],- 'useCases': [ "Footprint" ],- 'categories': [ "Content Analysis" ]+ 'flags': ["errorprone"],+ 'useCases': ["Footprint"],+ 'categories': ["Content Analysis"]}# Default options@@ -31,8 +31,8 @@ class sfp_binstring(SpiderFootPlugin):'maxfilesize': 1000000,'usedict': True,'fileexts': ['png', 'gif', 'jpg', 'jpeg', 'tiff', 'tif',- 'ico', 'flv', 'mp4', 'mp3', 'avi', 'mpg',- 'mpeg', 'dat', 'mov', 'swf', 'exe', 'bin'],+ 'ico', 'flv', 'mp4', 'mp3', 'avi', 'mpg',+ 'mpeg', 'dat', 'mov', 'swf', 'exe', 'bin'],'filterchars': '#}{|%^&*()=+,;[]~'}diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.pyindex 4a870efa..cbd06cec 100644--- a/modules/sfp_bitcoin.py+++ b/modules/sfp_bitcoin.py@@ -14,7 +14,7 @@import reimport codecsfrom hashlib import sha256-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_bitcoin(SpiderFootPlugin):"""Bitcoin Finder:Footprint,Investigate,Passive:Content Analysis::Identify bitcoin addresses in scraped webpages."""@@ -22,9 +22,9 @@ class sfp_bitcoin(SpiderFootPlugin):meta = {'name': "Bitcoin Finder",'summary': "Identify bitcoin addresses in scraped webpages.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_blockchain.py b/modules/sfp_blockchain.pyindex 90f528ae..ccafa82a 100644--- a/modules/sfp_blockchain.py+++ b/modules/sfp_blockchain.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_blockchain(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_blockchain(SpiderFootPlugin):meta = {'name': "Blockchain",'summary': "Queries blockchain.info to find the balance of identified bitcoin wallet addresses.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Public Registries" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Public Registries"],'dataSource': {'website': "https://www.blockchain.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -90,7 +90,7 @@ class sfp_blockchain(SpiderFootPlugin):data = json.loads(res['content'])balance = float(data[eventData]['final_balance']) / 100000000except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Noneevt = SpiderFootEvent("BITCOIN_BALANCE", str(balance) + " BTC", self.__name__, event)diff --git a/modules/sfp_blocklistde.py b/modules/sfp_blocklistde.pyindex f4a147e5..b7ae104a 100644--- a/modules/sfp_blocklistde.py+++ b/modules/sfp_blocklistde.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'blocklist.de List': {@@ -30,9 +30,9 @@ class sfp_blocklistde(SpiderFootPlugin):meta = {'name': "blocklist.de",'summary': "Check if a netblock or IP is malicious according to blocklist.de.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.blocklist.de/en/index.html",'model': "FREE_NOAUTH_UNLIMITED",@@ -126,8 +126,7 @@ class sfp_blocklistde(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_botscout.py b/modules/sfp_botscout.pyindex c2008fc2..e0165837 100644--- a/modules/sfp_botscout.py+++ b/modules/sfp_botscout.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_botscout(SpiderFootPlugin):@@ -20,9 +20,9 @@ class sfp_botscout(SpiderFootPlugin):meta = {'name': "BotScout",'summary': "Searches botscout.com's database of spam-bot IPs and e-mail addresses.",- 'flags': [ "apikey" ],- 'useCases': [ "Passive", "Investigate" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Passive", "Investigate"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://botscout.com/",'model': "FREE_NOAUTH_LIMITED",diff --git a/modules/sfp_botvrij.py b/modules/sfp_botvrij.pyindex d9f43bb6..32d2b004 100644--- a/modules/sfp_botvrij.py+++ b/modules/sfp_botvrij.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'botvrij.eu Domain Blocklist': {@@ -31,9 +31,9 @@ class sfp_botvrij(SpiderFootPlugin):meta = {'name': "botvrij.eu",'summary': "Check if a domain is malicious according to botvrij.eu.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ]+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"]}# Default options@@ -110,8 +110,7 @@ class sfp_botvrij(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_builtwith.py b/modules/sfp_builtwith.pyindex 3d87fc6c..32b22fa2 100644--- a/modules/sfp_builtwith.py+++ b/modules/sfp_builtwith.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_builtwith(SpiderFootPlugin):"""BuiltWith:Footprint,Investigate,Passive:Search Engines:apikey:Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more."""@@ -20,9 +20,9 @@ class sfp_builtwith(SpiderFootPlugin):meta = {'name': "BuiltWith",'summary': "Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://builtwith.com/",'model': "FREE_AUTH_LIMITED",@@ -80,21 +80,18 @@ class sfp_builtwith(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "DOMAIN_NAME" ]+ return ["DOMAIN_NAME"]# What events this module producesdef producedEvents(self):- return [ "INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",- "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME",- "CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID" ]+ return ["INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",+ "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME",+ "CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID"]def queryRelationships(self, t):- ret = None+ url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}"- url = "https://api.builtwith.com/rv1/api.json?LOOKUP=" + t + "&KEY=" + self.opts['api_key']-- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent="SpiderFoot")+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")if res['code'] == "404":return None@@ -103,20 +100,15 @@ class sfp_builtwith(SpiderFootPlugin):return Nonetry:- ret = json.loads(res['content'])['Relationships']+ return json.loads(res['content'])['Relationships']except Exception as e:- self.sf.error("Error processing JSON response from builtwith.com: " + str(e), False)+ self.sf.error(f"Error processing JSON response from builtwith.com: {e}", False)return None- return ret-def queryDomainInfo(self, t):- ret = None+ url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}"- url = "https://api.builtwith.com/v11/api.json?LOOKUP=" + t + "&KEY=" + self.opts['api_key']-- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent="SpiderFoot")+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")if res['code'] == "404":return None@@ -125,13 +117,11 @@ class sfp_builtwith(SpiderFootPlugin):return Nonetry:- ret = json.loads(res['content'])['Results'][0]+ return json.loads(res['content'])['Results'][0]except Exception as e:- self.sf.error("Error processing JSON response from builtwith.com: " + str(e), False)+ self.sf.error(f"Error processing JSON response from builtwith.com: {e}", False)return None- return ret-# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypediff --git a/modules/sfp_callername.py b/modules/sfp_callername.pyindex d8768db9..6fa80ad2 100644--- a/modules/sfp_callername.py+++ b/modules/sfp_callername.py@@ -13,7 +13,7 @@import reimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_callername(SpiderFootPlugin):"""CallerName:Footprint,Investigate,Passive:Real World::Lookup US phone number location and reputation information."""@@ -21,9 +21,9 @@ class sfp_callername(SpiderFootPlugin):meta = {'name': "CallerName",'summary': "Lookup US phone number location and reputation information.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Real World" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Real World"],'dataSource': {'website': "http://callername.com/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_censys.py b/modules/sfp_censys.pyindex 4a5bcb06..4269f379 100644--- a/modules/sfp_censys.py+++ b/modules/sfp_censys.py@@ -15,7 +15,7 @@ import base64from datetime import datetimeimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_censys(SpiderFootPlugin):"""Censys:Investigate,Passive:Search Engines:apikey:Obtain information from Censys.io"""@@ -23,9 +23,9 @@ class sfp_censys(SpiderFootPlugin):meta = {'name': "Censys",'summary': "Obtain information from Censys.io",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Search Engines" ]+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Search Engines"]}# Default options@@ -91,7 +91,7 @@ class sfp_censys(SpiderFootPlugin):# API rate limit: 0.4 actions/second (120.0 per 5 minute interval)time.sleep(self.opts['delay'])- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("Censys.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)self.errorState = Truereturn None@@ -103,7 +103,7 @@ class sfp_censys(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from Censys.io.", False)+ self.sf.error(f"Error processing JSON response from Censys.io: {e}", False)return None#print(str(info))diff --git a/modules/sfp_cinsscore.py b/modules/sfp_cinsscore.pyindex 746f2fed..f356abbc 100644--- a/modules/sfp_cinsscore.py+++ b/modules/sfp_cinsscore.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_cinsscore(SpiderFootPlugin):"""CINS Army List:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is malicious according to cinsscore.com's Army List."""@@ -19,9 +19,9 @@ class sfp_cinsscore(SpiderFootPlugin):meta = {'name': "CINS Army List",'summary': "Check if a netblock or IP address is malicious according to cinsscore.com's Army List.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ]+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"]}# Default optionsdiff --git a/modules/sfp_circllu.py b/modules/sfp_circllu.pyindex 65379d97..e50dff01 100644--- a/modules/sfp_circllu.py+++ b/modules/sfp_circllu.py@@ -15,7 +15,7 @@ import base64import reimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_circllu(SpiderFootPlugin):"""CIRCL.LU:Investigate,Passive:Reputation Systems:apikey:Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases."""@@ -23,9 +23,9 @@ class sfp_circllu(SpiderFootPlugin):meta = {'name': "CIRCL.LU",'summary': "Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.circl.lu/",'model': "FREE_AUTH_UNLIMITED",@@ -98,8 +98,6 @@ class sfp_circllu(SpiderFootPlugin):return ["IP_ADDRESS", "SSL_CERTIFICATE_ISSUED", "CO_HOSTED_SITE"]def query(self, qry, qtype):- ret = None-if self.errorState:return None@@ -118,7 +116,7 @@ class sfp_circllu(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=30,useragent="SpiderFoot", headers=headers)- if res['code'] not in [ "200", "201" ]:+ if res['code'] not in ["200", "201"]:self.sf.error("CIRCL.LU access seems to have been rejected or you have exceeded usage limits.", False)self.errorState = Truereturn None@@ -158,7 +156,7 @@ class sfp_circllu(SpiderFootPlugin):else:self.results[eventData] = True- if eventName in [ 'IP_ADDRESS', 'NETBLOCK_OWNER' ]:+ if eventName in ['IP_ADDRESS', 'NETBLOCK_OWNER']:# CIRCL.LU limit the maximum subnet size to 23# http://circl.lu/services/passive-ssl/if "/" in eventData:@@ -185,7 +183,7 @@ class sfp_circllu(SpiderFootPlugin):ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event)self.notifyListeners(ipe)for crt in j[ip]['subjects']:- r = re.findall(".*[\"\'](.+CN=([a-zA-Z0-9\-\*\.])+)[\"\'].*",+ r = re.findall(r".*[\"\'](.+CN=([a-zA-Z0-9\-\*\.])+)[\"\'].*",str(j[ip]['subjects'][crt]), re.IGNORECASE)if r:e = SpiderFootEvent("SSL_CERTIFICATE_ISSUED", r[0][0], self.__name__, ipe)@@ -193,7 +191,7 @@ class sfp_circllu(SpiderFootPlugin):except BaseException as e:self.sf.error("Invalid response returned from CIRCL.LU: " + str(e), False)- if eventName in [ 'IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME' ]:+ if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']:ret = self.query(eventData, "PDNS")if not ret:self.sf.info("No CIRCL.LU passive DNS data found for " + eventData)@@ -222,7 +220,7 @@ class sfp_circllu(SpiderFootPlugin):# We found a co-hostcohosts.append(rec['rrname'])- if eventName in [ "INTERNET_NAME", "DOMAIN_NAME" ]:+ if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]:# Record could be an A/CNAME of this entity, or something pointing to itif rec['rdata'] == eventData:if not self.getTarget().matches(rec['rrname']):diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.pyindex 02c4a152..26cbb35a 100644--- a/modules/sfp_citadel.py+++ b/modules/sfp_citadel.py@@ -12,7 +12,7 @@import jsonimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_citadel(SpiderFootPlugin):"""Leak-Lookup:Footprint,Investigate,Passive:Leaks, Dumps and Breaches:apikey:Searches Leak-Lookup.com's database of breaches."""@@ -20,9 +20,9 @@ class sfp_citadel(SpiderFootPlugin):meta = {'name': "Leak-Lookup",'summary': "Searches Leak-Lookup.com's database of breaches.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://leak-lookup.com/",'model': "FREE_AUTH_UNLIMITED",@@ -126,7 +126,7 @@ class sfp_citadel(SpiderFootPlugin):# Don't look up stuff twiceif eventData in self.results:- self.sf.debug("Skipping " + eventData + " as already searched.")+ self.sf.debug(f"Skipping {eventData}, already checked.")return Noneself.results[eventData] = True@@ -140,7 +140,7 @@ class sfp_citadel(SpiderFootPlugin):message = data.get('message')if error == 'true':- self.sf.error("Error encountered processing {}: {}".format( eventData, message ), False)+ self.sf.error(f"Error encountered processing {eventData}: {message}")if "MISSING API" in message:self.errorState = Truereturn None@@ -150,8 +150,8 @@ class sfp_citadel(SpiderFootPlugin):return Nonefor site in message:- self.sf.info("Found Leak-Lookup entry for {}: {}".format( eventData, site ) )- evt = SpiderFootEvent( "EMAILADDR_COMPROMISED", "{} [{}]".format( eventData, site ), self.__name__, event )+ self.sf.info(f"Found Leak-Lookup entry for {eventData}: {site}")+ evt = SpiderFootEvent("EMAILADDR_COMPROMISED", f"{eventData} [{site}]", self.__name__, event)self.notifyListeners(evt)# End of sfp_citadel classdiff --git a/modules/sfp_cleanbrowsing.py b/modules/sfp_cleanbrowsing.pyindex bd25d745..93c52c2b 100644--- a/modules/sfp_cleanbrowsing.py+++ b/modules/sfp_cleanbrowsing.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_cleanbrowsing(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_cleanbrowsing(SpiderFootPlugin):meta = {'name': "Cleanbrowsing.org",'summary': "Check if a host would be blocked by Cleanbrowsing.org DNS",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://cleanbrowsing.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -76,13 +76,13 @@ class sfp_cleanbrowsing(SpiderFootPlugin):def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "185.228.168.168", "185.228.168.169" ]+ res.nameservers = ["185.228.168.168", "185.228.168.169"]try:addrs = res.query(qaddr)self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -108,7 +108,7 @@ class sfp_cleanbrowsing(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:+ except BaseException:return Noneif resolved:diff --git a/modules/sfp_cleantalk.py b/modules/sfp_cleantalk.pyindex 1fda22e7..0b1aaa34 100644--- a/modules/sfp_cleantalk.py+++ b/modules/sfp_cleantalk.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_cleantalk(SpiderFootPlugin):"""CleanTalk Spam List:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is on CleanTalk.org's spam IP list."""@@ -19,9 +19,9 @@ class sfp_cleantalk(SpiderFootPlugin):meta = {'name': "CleanTalk Spam List",'summary': "Check if a netblock or IP address is on CleanTalk.org's spam IP list.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://cleantalk.org",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_clearbit.py b/modules/sfp_clearbit.pyindex b2f582b4..e71b2e3f 100644--- a/modules/sfp_clearbit.py+++ b/modules/sfp_clearbit.py@@ -11,7 +11,7 @@import jsonimport base64-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_clearbit(SpiderFootPlugin):"""Clearbit:Footprint,Investigate,Passive:Search Engines:apikey:Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com."""@@ -19,9 +19,9 @@ class sfp_clearbit(SpiderFootPlugin):meta = {'name': "Clearbit",'summary': "Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://clearbit.com/",'model': "FREE_AUTH_LIMITED",@@ -77,12 +77,12 @@ class sfp_clearbit(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "EMAILADDR" ]+ return ["EMAILADDR"]# What events this module producesdef producedEvents(self):- return [ "RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS",- "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC" ]+ return ["RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS",+ "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC"]def query(self, t):ret = None@@ -97,19 +97,16 @@ class sfp_clearbit(SpiderFootPlugin):'Authorization': "Basic " + token.decode('utf-8')}- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent="SpiderFoot", headers=headers)+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers)if res['code'] != "200":- self.sf.error("Return code indicates no results or potential API key failure or exceeded limits.",- False)+ self.sf.error("Return code indicates no results or potential API key failure or exceeded limits.", False)return Nonetry:ret = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from clearbit.io: " + \- str(e), False)+ self.sf.error(f"Error processing JSON response from clearbit.io: {e}", False)return Nonereturn retdiff --git a/modules/sfp_cloudflaredns.py b/modules/sfp_cloudflaredns.pyindex 35ddf762..f3604c64 100644--- a/modules/sfp_cloudflaredns.py+++ b/modules/sfp_cloudflaredns.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_cloudflaredns(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_cloudflaredns(SpiderFootPlugin):meta = {'name': "CloudFlare Malware DNS",'summary': "Check if a host would be blocked by CloudFlare Malware-blocking DNS",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.cloudflare.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -69,13 +69,13 @@ class sfp_cloudflaredns(SpiderFootPlugin):def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "1.1.1.2", "1.0.0.2" ]+ res.nameservers = ["1.1.1.2", "1.0.0.2"]try:addrs = res.query(qaddr)self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -106,7 +106,7 @@ class sfp_cloudflaredns(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:+ except BaseException:return Noneif resolved:diff --git a/modules/sfp_coinblocker.py b/modules/sfp_coinblocker.pyindex 7c6a8c81..4a7744b7 100644--- a/modules/sfp_coinblocker.py+++ b/modules/sfp_coinblocker.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'CoinBlocker IP List': {@@ -36,9 +36,9 @@ class sfp_coinblocker(SpiderFootPlugin):meta = {'name': "CoinBlocker Lists",'summary': "Check if a host/domain or IP appears on CoinBlocker lists.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://zerodot1.gitlab.io/CoinBlockerListsWeb/",'model': "FREE_NOAUTH_UNLIMITED",@@ -138,8 +138,7 @@ class sfp_coinblocker(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_commoncrawl.py b/modules/sfp_commoncrawl.pyindex b941bc15..cf5edac9 100644--- a/modules/sfp_commoncrawl.py+++ b/modules/sfp_commoncrawl.py@@ -14,7 +14,7 @@import reimport json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_commoncrawl(SpiderFootPlugin):@@ -23,9 +23,9 @@ class sfp_commoncrawl(SpiderFootPlugin):meta = {'name': "CommonCrawl",'summary': "Searches for URLs found through CommonCrawl.org.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "http://commoncrawl.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -74,7 +74,7 @@ class sfp_commoncrawl(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=60,useragent="SpiderFoot")- if res['code'] in [ "400", "401", "402", "403", "404" ]:+ if res['code'] in ["400", "401", "402", "403", "404"]:self.sf.error("CommonCrawl search doesn't seem to be available.", False)self.errorState = Truereturn None@@ -93,7 +93,7 @@ class sfp_commoncrawl(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=60,useragent="SpiderFoot")- if res['code'] in [ "400", "401", "402", "403", "404" ]:+ if res['code'] in ["400", "401", "402", "403", "404"]:self.sf.error("CommonCrawl index collection doesn't seem to be available.", False)self.errorState = Truereturn list()@@ -103,8 +103,7 @@ class sfp_commoncrawl(SpiderFootPlugin):self.errorState = Truereturn list()- indexes = re.findall(".*(CC-MAIN-\d+-\d+).*", res['content'])- highest = 0+ indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", res['content'])indexlist = dict()for m in indexes:ms = m.replace("CC-MAIN-", "").replace("-", "")diff --git a/modules/sfp_comodo.py b/modules/sfp_comodo.pyindex 94704fc6..008e7b1b 100644--- a/modules/sfp_comodo.py+++ b/modules/sfp_comodo.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_comodo(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_comodo(SpiderFootPlugin):meta = {'name': "Comodo",'summary': "Check if a host would be blocked by Comodo DNS",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.comodo.com/secure-dns/",'model': "FREE_NOAUTH_UNLIMITED",@@ -72,13 +72,13 @@ class sfp_comodo(SpiderFootPlugin):def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "8.26.56.26", "8.20.247.20" ]+ res.nameservers = ["8.26.56.26", "8.20.247.20"]try:addrs = res.query(qaddr)self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -104,7 +104,7 @@ class sfp_comodo(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:+ except BaseException:return Noneif resolved:diff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.pyindex e667b2db..29f58829 100644--- a/modules/sfp_cookie.py+++ b/modules/sfp_cookie.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_cookie(SpiderFootPlugin):"""Cookie Extractor:Footprint,Investigate,Passive:Content Analysis::Extract Cookies from HTTP headers."""@@ -19,9 +19,9 @@ class sfp_cookie(SpiderFootPlugin):meta = {'name': "Cookie Extractor",'summary': "Extract Cookies from HTTP headers.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default options@@ -69,7 +69,7 @@ class sfp_cookie(SpiderFootPlugin):jdata = json.loads(eventData)if jdata == None:return None- except BaseException as e:+ except BaseException:self.sf.error("Received HTTP headers from another module in an unexpected format.", False)return Nonediff --git a/modules/sfp_countryname.py b/modules/sfp_countryname.pyindex 1ba926f2..f5275bd4 100644--- a/modules/sfp_countryname.py+++ b/modules/sfp_countryname.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport phonenumbersfrom phonenumbers.phonenumberutil import region_code_for_country_code@@ -22,28 +22,28 @@ class sfp_countryname(SpiderFootPlugin):meta = {'name': "Country Name Extractor",'summary': "Identify country names in any obtained data.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsopts = {# options specific to this module- 'cohosted' : True,- 'affiliate' : True,- 'noncountrytld' : True,- 'noncountrytlddefault' : "United States",- 'similardomain' : False,+ 'cohosted': True,+ 'affiliate': True,+ 'noncountrytld': True,+ 'noncountrytlddefault': "United States",+ 'similardomain': False,}# Option descriptionsoptdescs = {- 'cohosted' : "Obtain country name from co-hosted sites",- 'affiliate' : "Obtain country name from affiliate sites",- 'noncountrytld' : "Parse TLDs not associated with any country as default country domains",- 'noncountrytlddefault' : "Default country name for TLDs not associated with any country(.com, .net, .org, .gov, .mil)",- 'similardomain' : "Obtain country name from similar domains"+ 'cohosted': "Obtain country name from co-hosted sites",+ 'affiliate': "Obtain country name from affiliate sites",+ 'noncountrytld': "Parse TLDs not associated with any country as default country domains",+ 'noncountrytlddefault': "Default country name for TLDs not associated with any country(.com, .net, .org, .gov, .mil)",+ 'similardomain': "Obtain country name from similar domains"}results = None@@ -62,93 +62,263 @@ class sfp_countryname(SpiderFootPlugin):# Dictionary of country codes and country namesabbvCountryCodes = {- "AF" : "Afghanistan", "AX" : "Aland Islands", "AL" : "Albania",- "DZ" : "Algeria", "AS" : "American Samoa", "AD" : "Andorra",- "AO" : "Angola", "AI" : "Anguilla", "AQ" : "Antarctica",- "AG" : "Antigua and Barbuda", "AR" : "Argentina", "AM" : "Armenia",- "AW" : "Aruba", "AU" : "Australia", "AT" : "Austria",- "AZ" : "Azerbaijan", "BS" : "Bahamas", "BH" : "Bahrain",- "BD" : "Bangladesh", "BB" : "Barbados", "BY" : "Belarus",- "BE" : "Belgium", "BZ" : "Belize", "BJ" : "Benin",- "BM" : "Bermuda", "BT" : "Bhutan", "BO" : "Bolivia",- "BQ" : "Bonaire, Saint Eustatius and Saba", "BA" : "Bosnia and Herzegovina", "BW" : "Botswana",- "BV" : "Bouvet Island", "BR" : "Brazil", "IO" : "British Indian Ocean Territory",- "VG" : "British Virgin Islands", "BN" : "Brunei", "BG" : "Bulgaria",- "BF" : "Burkina Faso", "BI" : "Burundi", "KH" : "Cambodia",- "CM" : "Cameroon", "CA" : "Canada", "CV" : "Cape Verde",- "KY" : "Cayman Islands", "CF" : "Central African Republic", "TD" : "Chad",- "CL" : "Chile", "CN" : "China", "CX" : "Christmas Island",- "CC" : "Cocos Islands", "CO" : "Colombia", "KM" : "Comoros",- "CK" : "Cook Islands", "CR" : "Costa Rica", "HR" : "Croatia",- "CU" : "Cuba", "CW" : "Curacao", "CY" : "Cyprus",- "CZ" : "Czech Republic", "CD" : "Democratic Republic of the Congo", "DK" : "Denmark",- "DJ" : "Djibouti", "DM" : "Dominica", "DO" : "Dominican Republic",- "TL" : "East Timor", "EC" : "Ecuador", "EG" : "Egypt",- "SV" : "El Salvador", "GQ" : "Equatorial Guinea", "ER" : "Eritrea",- "EE" : "Estonia", "ET" : "Ethiopia", "FK" : "Falkland Islands",- "FO" : "Faroe Islands", "FJ" : "Fiji", "FI" : "Finland",- "FR" : "France", "GF" : "French Guiana", "PF" : "French Polynesia",- "TF" : "French Southern Territories", "GA" : "Gabon", "GM" : "Gambia",- "GE" : "Georgia", "DE" : "Germany", "GH" : "Ghana",- "GI" : "Gibraltar", "GR" : "Greece", "GL" : "Greenland",- "GD" : "Grenada", "GP" : "Guadeloupe", "GU" : "Guam",- "GT" : "Guatemala", "GG" : "Guernsey", "GN" : "Guinea",- "GW" : "Guinea-Bissau", "GY" : "Guyana", "HT" : "Haiti",- "HM" : "Heard Island and McDonald Islands", "HN" : "Honduras", "HK" : "Hong Kong",- "HU" : "Hungary", "IS" : "Iceland", "IN" : "India",- "ID" : "Indonesia", "IR" : "Iran", "IQ" : "Iraq",- "IE" : "Ireland", "IM" : "Isle of Man", "IL" : "Israel",- "IT" : "Italy", "CI" : "Ivory Coast", "JM" : "Jamaica",- "JP" : "Japan", "JE" : "Jersey", "JO" : "Jordan",- "KZ" : "Kazakhstan", "KE" : "Kenya", "KI" : "Kiribati",- "XK" : "Kosovo", "KW" : "Kuwait", "KG" : "Kyrgyzstan",- "LA" : "Laos", "LV" : "Latvia", "LB" : "Lebanon",- "LS" : "Lesotho", "LR" : "Liberia", "LY" : "Libya",- "LI" : "Liechtenstein", "LT" : "Lithuania", "LU" : "Luxembourg",- "MO" : "Macao", "MK" : "Macedonia", "MG" : "Madagascar",- "MW" : "Malawi", "MY" : "Malaysia", "MV" : "Maldives",- "ML" : "Mali", "MT" : "Malta", "MH" : "Marshall Islands",- "MQ" : "Martinique", "MR" : "Mauritania", "MU" : "Mauritius",- "YT" : "Mayotte", "MX" : "Mexico", "FM" : "Micronesia",- "MD" : "Moldova", "MC" : "Monaco", "MN" : "Mongolia",- "ME" : "Montenegro", "MS" : "Montserrat", "MA" : "Morocco",- "MZ" : "Mozambique", "MM" : "Myanmar", "NA" : "Namibia",- "NR" : "Nauru", "NP" : "Nepal", "NL" : "Netherlands",- "AN" : "Netherlands Antilles", "NC" : "New Caledonia", "NZ" : "New Zealand",- "NI" : "Nicaragua", "NE" : "Niger", "NG" : "Nigeria",- "NU" : "Niue", "NF" : "Norfolk Island", "KP" : "North Korea",- "MP" : "Northern Mariana Islands", "NO" : "Norway", "OM" : "Oman",- "PK" : "Pakistan", "PW" : "Palau", "PS" : "Palestinian Territory",- "PA" : "Panama", "PG" : "Papua New Guinea", "PY" : "Paraguay",- "PE" : "Peru", "PH" : "Philippines", "PN" : "Pitcairn",- "PL" : "Poland", "PT" : "Portugal", "PR" : "Puerto Rico",- "QA" : "Qatar", "CG" : "Republic of the Congo", "RE" : "Reunion",- "RO" : "Romania", "RU" : "Russia", "RW" : "Rwanda",- "BL" : "Saint Barthelemy", "SH" : "Saint Helena", "KN" : "Saint Kitts and Nevis",- "LC" : "Saint Lucia", "MF" : "Saint Martin", "PM" : "Saint Pierre and Miquelon",- "VC" : "Saint Vincent and the Grenadines", "WS" : "Samoa", "SM" : "San Marino",- "ST" : "Sao Tome and Principe", "SA" : "Saudi Arabia", "SN" : "Senegal",- "RS" : "Serbia", "CS" : "Serbia and Montenegro", "SC" : "Seychelles",- "SL" : "Sierra Leone", "SG" : "Singapore", "SX" : "Sint Maarten",- "SK" : "Slovakia", "SI" : "Slovenia", "SB" : "Solomon Islands",- "SO" : "Somalia", "ZA" : "South Africa", "GS" : "South Georgia and the South Sandwich Islands",- "KR" : "South Korea", "SS" : "South Sudan", "ES" : "Spain",- "LK" : "Sri Lanka", "SD" : "Sudan", "SR" : "Suriname",- "SJ" : "Svalbard and Jan Mayen", "SZ" : "Swaziland", "SE" : "Sweden",- "CH" : "Switzerland", "SY" : "Syria", "TW" : "Taiwan",- "TJ" : "Tajikistan", "TZ" : "Tanzania", "TH" : "Thailand",- "TG" : "Togo", "TK" : "Tokelau", "TO" : "Tonga",- "TT" : "Trinidad and Tobago", "TN" : "Tunisia", "TR" : "Turkey",- "TM" : "Turkmenistan", "TC" : "Turks and Caicos Islands", "TV" : "Tuvalu",- "VI" : "U.S. Virgin Islands", "UG" : "Uganda", "UA" : "Ukraine",- "AE" : "United Arab Emirates", "GB" : "United Kingdom", "US" : "United States",- "UM" : "United States Minor Outlying Islands", "UY" : "Uruguay", "UZ" : "Uzbekistan",- "VU" : "Vanuatu", "VA" : "Vatican", "VE" : "Venezuela",- "VN" : "Vietnam", "WF" : "Wallis and Futuna", "EH" : "Western Sahara",- "YE" : "Yemen", "ZM" : "Zambia", "ZW" : "Zimbabwe",+ "AF": "Afghanistan",+ "AX": "Aland Islands",+ "AL": "Albania",+ "DZ": "Algeria",+ "AS": "American Samoa",+ "AD": "Andorra",+ "AO": "Angola",+ "AI": "Anguilla",+ "AQ": "Antarctica",+ "AG": "Antigua and Barbuda",+ "AR": "Argentina",+ "AM": "Armenia",+ "AW": "Aruba",+ "AU": "Australia",+ "AT": "Austria",+ "AZ": "Azerbaijan",+ "BS": "Bahamas",+ "BH": "Bahrain",+ "BD": "Bangladesh",+ "BB": "Barbados",+ "BY": "Belarus",+ "BE": "Belgium",+ "BZ": "Belize",+ "BJ": "Benin",+ "BM": "Bermuda",+ "BT": "Bhutan",+ "BO": "Bolivia",+ "BQ": "Bonaire, Saint Eustatius and Saba",+ "BA": "Bosnia and Herzegovina",+ "BW": "Botswana",+ "BV": "Bouvet Island",+ "BR": "Brazil",+ "IO": "British Indian Ocean Territory",+ "VG": "British Virgin Islands",+ "BN": "Brunei",+ "BG": "Bulgaria",+ "BF": "Burkina Faso",+ "BI": "Burundi",+ "KH": "Cambodia",+ "CM": "Cameroon",+ "CA": "Canada",+ "CV": "Cape Verde",+ "KY": "Cayman Islands",+ "CF": "Central African Republic",+ "TD": "Chad",+ "CL": "Chile",+ "CN": "China",+ "CX": "Christmas Island",+ "CC": "Cocos Islands",+ "CO": "Colombia",+ "KM": "Comoros",+ "CK": "Cook Islands",+ "CR": "Costa Rica",+ "HR": "Croatia",+ "CU": "Cuba",+ "CW": "Curacao",+ "CY": "Cyprus",+ "CZ": "Czech Republic",+ "CD": "Democratic Republic of the Congo",+ "DK": "Denmark",+ "DJ": "Djibouti",+ "DM": "Dominica",+ "DO": "Dominican Republic",+ "TL": "East Timor",+ "EC": "Ecuador",+ "EG": "Egypt",+ "SV": "El Salvador",+ "GQ": "Equatorial Guinea",+ "ER": "Eritrea",+ "EE": "Estonia",+ "ET": "Ethiopia",+ "FK": "Falkland Islands",+ "FO": "Faroe Islands",+ "FJ": "Fiji",+ "FI": "Finland",+ "FR": "France",+ "GF": "French Guiana",+ "PF": "French Polynesia",+ "TF": "French Southern Territories",+ "GA": "Gabon",+ "GM": "Gambia",+ "GE": "Georgia",+ "DE": "Germany",+ "GH": "Ghana",+ "GI": "Gibraltar",+ "GR": "Greece",+ "GL": "Greenland",+ "GD": "Grenada",+ "GP": "Guadeloupe",+ "GU": "Guam",+ "GT": "Guatemala",+ "GG": "Guernsey",+ "GN": "Guinea",+ "GW": "Guinea-Bissau",+ "GY": "Guyana",+ "HT": "Haiti",+ "HM": "Heard Island and McDonald Islands",+ "HN": "Honduras",+ "HK": "Hong Kong",+ "HU": "Hungary",+ "IS": "Iceland",+ "IN": "India",+ "ID": "Indonesia",+ "IR": "Iran",+ "IQ": "Iraq",+ "IE": "Ireland",+ "IM": "Isle of Man",+ "IL": "Israel",+ "IT": "Italy",+ "CI": "Ivory Coast",+ "JM": "Jamaica",+ "JP": "Japan",+ "JE": "Jersey",+ "JO": "Jordan",+ "KZ": "Kazakhstan",+ "KE": "Kenya",+ "KI": "Kiribati",+ "XK": "Kosovo",+ "KW": "Kuwait",+ "KG": "Kyrgyzstan",+ "LA": "Laos",+ "LV": "Latvia",+ "LB": "Lebanon",+ "LS": "Lesotho",+ "LR": "Liberia",+ "LY": "Libya",+ "LI": "Liechtenstein",+ "LT": "Lithuania",+ "LU": "Luxembourg",+ "MO": "Macao",+ "MK": "Macedonia",+ "MG": "Madagascar",+ "MW": "Malawi",+ "MY": "Malaysia",+ "MV": "Maldives",+ "ML": "Mali",+ "MT": "Malta",+ "MH": "Marshall Islands",+ "MQ": "Martinique",+ "MR": "Mauritania",+ "MU": "Mauritius",+ "YT": "Mayotte",+ "MX": "Mexico",+ "FM": "Micronesia",+ "MD": "Moldova",+ "MC": "Monaco",+ "MN": "Mongolia",+ "ME": "Montenegro",+ "MS": "Montserrat",+ "MA": "Morocco",+ "MZ": "Mozambique",+ "MM": "Myanmar",+ "NA": "Namibia",+ "NR": "Nauru",+ "NP": "Nepal",+ "NL": "Netherlands",+ "AN": "Netherlands Antilles",+ "NC": "New Caledonia",+ "NZ": "New Zealand",+ "NI": "Nicaragua",+ "NE": "Niger",+ "NG": "Nigeria",+ "NU": "Niue",+ "NF": "Norfolk Island",+ "KP": "North Korea",+ "MP": "Northern Mariana Islands",+ "NO": "Norway",+ "OM": "Oman",+ "PK": "Pakistan",+ "PW": "Palau",+ "PS": "Palestinian Territory",+ "PA": "Panama",+ "PG": "Papua New Guinea",+ "PY": "Paraguay",+ "PE": "Peru",+ "PH": "Philippines",+ "PN": "Pitcairn",+ "PL": "Poland",+ "PT": "Portugal",+ "PR": "Puerto Rico",+ "QA": "Qatar",+ "CG": "Republic of the Congo",+ "RE": "Reunion",+ "RO": "Romania",+ "RU": "Russia",+ "RW": "Rwanda",+ "BL": "Saint Barthelemy",+ "SH": "Saint Helena",+ "KN": "Saint Kitts and Nevis",+ "LC": "Saint Lucia",+ "MF": "Saint Martin",+ "PM": "Saint Pierre and Miquelon",+ "VC": "Saint Vincent and the Grenadines",+ "WS": "Samoa",+ "SM": "San Marino",+ "ST": "Sao Tome and Principe",+ "SA": "Saudi Arabia",+ "SN": "Senegal",+ "RS": "Serbia",+ "CS": "Serbia and Montenegro",+ "SC": "Seychelles",+ "SL": "Sierra Leone",+ "SG": "Singapore",+ "SX": "Sint Maarten",+ "SK": "Slovakia",+ "SI": "Slovenia",+ "SB": "Solomon Islands",+ "SO": "Somalia",+ "ZA": "South Africa",+ "GS": "South Georgia and the South Sandwich Islands",+ "KR": "South Korea",+ "SS": "South Sudan",+ "ES": "Spain",+ "LK": "Sri Lanka",+ "SD": "Sudan",+ "SR": "Suriname",+ "SJ": "Svalbard and Jan Mayen",+ "SZ": "Swaziland",+ "SE": "Sweden",+ "CH": "Switzerland",+ "SY": "Syria",+ "TW": "Taiwan",+ "TJ": "Tajikistan",+ "TZ": "Tanzania",+ "TH": "Thailand",+ "TG": "Togo",+ "TK": "Tokelau",+ "TO": "Tonga",+ "TT": "Trinidad and Tobago",+ "TN": "Tunisia",+ "TR": "Turkey",+ "TM": "Turkmenistan",+ "TC": "Turks and Caicos Islands",+ "TV": "Tuvalu",+ "VI": "U.S. Virgin Islands",+ "UG": "Uganda",+ "UA": "Ukraine",+ "AE": "United Arab Emirates",+ "GB": "United Kingdom",+ "US": "United States",+ "UM": "United States Minor Outlying Islands",+ "UY": "Uruguay",+ "UZ": "Uzbekistan",+ "VU": "Vanuatu",+ "VA": "Vatican",+ "VE": "Venezuela",+ "VN": "Vietnam",+ "WF": "Wallis and Futuna",+ "EH": "Western Sahara",+ "YE": "Yemen",+ "ZM": "Zambia",+ "ZW": "Zimbabwe",# Below are not country codes but recognized as TLDs- "AC" : "Ascension Island", "EU" : "European Union", "SU" : "Soviet Union",- "UK" : "United Kingdom"+ "AC": "Ascension Island",+ "EU": "European Union",+ "SU": "Soviet Union",+ "UK": "United Kingdom"}return abbvCountryCodes@@ -241,7 +411,7 @@ class sfp_countryname(SpiderFootPlugin):# Look for country name in source data# Spaces are not included since New Jersey and others# will get interpreted as Jersey, etc.- matchCountries = re.findall(r"[,'\"\:\=\[\(\[\n\t\r\.] ?" + countryName + "[,'\"\:\=\[\(\[\n\t\r\.]", srcData, re.IGNORECASE)+ matchCountries = re.findall(r"[,'\"\:\=\[\(\[\n\t\r\.] ?" + countryName + r"[,'\"\:\=\[\(\[\n\t\r\.]", srcData, re.IGNORECASE)if len(matchCountries) > 0:# Get country name from first index of list@@ -306,7 +476,7 @@ class sfp_countryname(SpiderFootPlugin):# Process the event data based on incoming event typeif eventName == "PHONE_NUMBER":countryNames.append(self.detectCountryFromPhone(eventData))- elif eventName == "DOMAIN_NAME" or (eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]) or(eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]) or (eventName == "SIMILARDOMAIN" and self.opts["similardomain"]):+ elif eventName == "DOMAIN_NAME" or (eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]) or (eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]) or (eventName == "SIMILARDOMAIN" and self.opts["similardomain"]):countryNames.append(self.detectCountryFromTLD(eventData))elif eventName == "IBAN_NUMBER":countryNames.append(self.detectCountryFromIBAN(eventData))diff --git a/modules/sfp_creditcard.py b/modules/sfp_creditcard.pyindex 16220ab0..75b6dfc7 100644--- a/modules/sfp_creditcard.py+++ b/modules/sfp_creditcard.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_creditcard(SpiderFootPlugin):"""Credit Card Number Extractor:Footprint,Investigate,Passive:Content Analysis::Identify Credit Card Numbers in any data"""@@ -19,9 +19,9 @@ class sfp_creditcard(SpiderFootPlugin):meta = {'name': "Credit Card Number Extractor",'summary': "Identify Credit Card Numbers in any data",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default options.@@ -55,7 +55,7 @@ class sfp_creditcard(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "DARKNET_MENTION_CONTENT", "LEAKSITE_CONTENT" ]+ return ["DARKNET_MENTION_CONTENT", "LEAKSITE_CONTENT"]# What events this module producesdef producedEvents(self):diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.pyindex b7717724..897a9076 100644--- a/modules/sfp_crossref.py+++ b/modules/sfp_crossref.py@@ -15,7 +15,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_crossref(SpiderFootPlugin):"""Cross-Referencer:Footprint:Crawling and Scanning::Identify whether other domains are associated ('Affiliates') of the target."""@@ -23,9 +23,9 @@ class sfp_crossref(SpiderFootPlugin):meta = {'name': "Cross-Referencer",'summary': "Identify whether other domains are associated ('Affiliates') of the target.",- 'flags': [ "" ],- 'useCases': [ "Footprint" ],- 'categories': [ "Crawling and Scanning" ]+ 'flags': [""],+ 'useCases': ["Footprint"],+ 'categories': ["Crawling and Scanning"]}# Default options@@ -97,7 +97,7 @@ class sfp_crossref(SpiderFootPlugin):matched = Falsefor name in self.getTarget().getNames():# Search for mentions of our host/domain in the external site's data- pat = re.compile("([\.\'\/\"\ ]" + name + "[\.\'\/\"\ ])", re.IGNORECASE)+ pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\.\'\/\"\ ])", re.IGNORECASE)matches = re.findall(pat, res['content'])if len(matches) > 0:@@ -122,7 +122,7 @@ class sfp_crossref(SpiderFootPlugin):verify=False)if res['content'] is not None:for name in self.getTarget().getNames():- pat = re.compile("([\.\'\/\"\ ]" + name + "[\'\/\"\ ])",+ pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\'\/\"\ ])",re.IGNORECASE)matches = re.findall(pat, res['content'])diff --git a/modules/sfp_crt.py b/modules/sfp_crt.pyindex 26576334..4c80ba11 100644--- a/modules/sfp_crt.py+++ b/modules/sfp_crt.py@@ -13,7 +13,7 @@import jsonimport urllib.parse-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_crt(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_crt(SpiderFootPlugin):meta = {'name': "Certificate Transparency",'summary': "Gather hostnames from historical certificates in crt.sh.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://crt.sh/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_customfeed.py b/modules/sfp_customfeed.pyindex 5bc20887..9bf7c852 100644--- a/modules/sfp_customfeed.py+++ b/modules/sfp_customfeed.py@@ -14,7 +14,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'Custom Threat Data': {@@ -30,9 +30,9 @@ class sfp_customfeed(SpiderFootPlugin):meta = {'name': "Custom Threat Feed",'summary': "Check if a host/domain, netblock, ASN or IP is malicious according to your custom feed.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ]+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"]}# Default options@@ -112,8 +112,7 @@ class sfp_customfeed(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.pyindex fc2c4624..09a122a2 100644--- a/modules/sfp_cybercrimetracker.py+++ b/modules/sfp_cybercrimetracker.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'cybercrime-tracker.net Malicious Submissions': {@@ -31,9 +31,9 @@ class sfp_cybercrimetracker(SpiderFootPlugin):meta = {'name': "cybercrime-tracker.net",'summary': "Check if a host/domain or IP is malicious according to cybercrime-tracker.net.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://cybercrime-tracker.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -122,8 +122,7 @@ class sfp_cybercrimetracker(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_darksearch.py b/modules/sfp_darksearch.pyindex bc81cc91..34d05eea 100644--- a/modules/sfp_darksearch.py+++ b/modules/sfp_darksearch.py@@ -14,7 +14,7 @@import jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_darksearch(SpiderFootPlugin):"""Darksearch:Footprint,Investigate:Search Engines::Search the Darksearch.io Tor search engine for mentions of the target domain."""@@ -22,9 +22,9 @@ class sfp_darksearch(SpiderFootPlugin):meta = {'name': "Darksearch",'summary': "Search the Darksearch.io Tor search engine for mentions of the target domain.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://darksearch.io/",'model': "FREE_NOAUTH_UNLIMITED",@@ -181,7 +181,7 @@ class sfp_darksearch(SpiderFootPlugin):try:startIndex = res['content'].index(eventData) - 120endIndex = startIndex + len(eventData) + 240- except BaseException as e:+ except BaseException:self.sf.debug("String not found in content.")continuediff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.pyindex af768492..90d25a03 100644--- a/modules/sfp_digitaloceanspace.py+++ b/modules/sfp_digitaloceanspace.py@@ -13,7 +13,7 @@import threadingimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_digitaloceanspace(SpiderFootPlugin):"""Digital Ocean Space Finder:Footprint,Passive:Crawling and Scanning::Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents."""@@ -21,9 +21,9 @@ class sfp_digitaloceanspace(SpiderFootPlugin):meta = {'name': "Digital Ocean Space Finder",'summary': "Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Crawling and Scanning" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Crawling and Scanning"],'dataSource': {'website': "https://www.digitalocean.com/products/spaces/",'model': "FREE_NOAUTH_UNLIMITED"@@ -70,7 +70,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):def checkSite(self, url):res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)- if res['code'] not in [ "301", "302", "200" ] and \+ if res['code'] not in ["301", "302", "200"] and \(res['content'] is None or "NoSuchBucket" in res['content']):self.sf.debug("Not a valid bucket: " + url)else:@@ -82,7 +82,6 @@ class sfp_digitaloceanspace(SpiderFootPlugin):self.s3results[url] = 0def threadSites(self, siteList):- ret = list()self.s3results = dict()running = Truei = 0@@ -156,7 +155,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):self.notifyListeners(evt)return None- targets = [ eventData.replace('.', '') ]+ targets = [eventData.replace('.', '')]kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])if kw:targets.append(kw)diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.pyindex e2fd0194..f61d80d5 100644--- a/modules/sfp_dnsbrute.py+++ b/modules/sfp_dnsbrute.py@@ -14,7 +14,7 @@import randomimport threadingimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_dnsbrute(SpiderFootPlugin):"""DNS Brute-forcer:Footprint,Investigate:DNS::Attempts to identify hostnames through brute-forcing common names and iterations."""@@ -22,9 +22,9 @@ class sfp_dnsbrute(SpiderFootPlugin):meta = {'name': "DNS Brute-forcer",'summary': "Attempts to identify hostnames through brute-forcing common names and iterations.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["DNS"]}# Default options@@ -98,7 +98,7 @@ class sfp_dnsbrute(SpiderFootPlugin):if addrs:with self.lock:self.hostResults[name] = True- except BaseException as e:+ except BaseException:with self.lock:self.hostResults[name] = False@@ -145,7 +145,6 @@ class sfp_dnsbrute(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataeventDataHash = self.sf.hashstring(eventData)- parentEvent = eventself.sf.debug(f"Received event, {eventName}, from {srcModuleName}")diff --git a/modules/sfp_dnscommonsrv.py b/modules/sfp_dnscommonsrv.pyindex 56c35ac7..3ed27d9b 100644--- a/modules/sfp_dnscommonsrv.py+++ b/modules/sfp_dnscommonsrv.py@@ -20,9 +20,9 @@ class sfp_dnscommonsrv(SpiderFootPlugin):meta = {'name': "DNS Common SRV",'summary': "Attempts to identify hostnames through common SRV.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["DNS"]}# Default options@@ -33,44 +33,46 @@ class sfp_dnscommonsrv(SpiderFootPlugin):events = None- commonsrv = [ # LDAP/Kerberos, used for Active Directory- # https://technet.microsoft.com/en-us/library/cc961719.aspx- '_ldap._tcp',- '_gc._msdcs',- '_ldap._tcp.pdc._msdcs',- '_ldap._tcp.gc._msdcs',- '_kerberos._tcp.dc._msdcs',- '_kerberos._tcp',- '_kerberos._udp',- '_kerberos-master._tcp',- '_kerberos-master._udp',- '_kpasswd._tcp',- '_kpasswd._udp',- '_ntp._udp',-- # SIP- '_sip._tcp',- '_sip._udp',- '_sip._tls',- '_sips._tcp',-- # STUN- # https://tools.ietf.org/html/rfc5389- '_stun._tcp',- '_stun._udp',- '_stuns._tcp',-- # TURN- # https://tools.ietf.org/html/rfc5928- '_turn._tcp',- '_turn._udp',- '_turns._tcp',-- # XMPP- # http://xmpp.org/rfcs/rfc6120.html- '_jabber._tcp',- '_xmpp-client._tcp',- '_xmpp-server._tcp']+ commonsrv = [+ # LDAP/Kerberos, used for Active Directory+ # https://technet.microsoft.com/en-us/library/cc961719.aspx+ '_ldap._tcp',+ '_gc._msdcs',+ '_ldap._tcp.pdc._msdcs',+ '_ldap._tcp.gc._msdcs',+ '_kerberos._tcp.dc._msdcs',+ '_kerberos._tcp',+ '_kerberos._udp',+ '_kerberos-master._tcp',+ '_kerberos-master._udp',+ '_kpasswd._tcp',+ '_kpasswd._udp',+ '_ntp._udp',++ # SIP+ '_sip._tcp',+ '_sip._udp',+ '_sip._tls',+ '_sips._tcp',++ # STUN+ # https://tools.ietf.org/html/rfc5389+ '_stun._tcp',+ '_stun._udp',+ '_stuns._tcp',++ # TURN+ # https://tools.ietf.org/html/rfc5928+ '_turn._tcp',+ '_turn._udp',+ '_turns._tcp',++ # XMPP+ # http://xmpp.org/rfcs/rfc6120.html+ '_jabber._tcp',+ '_xmpp-client._tcp',+ '_xmpp-server._tcp'+ ]def setup(self, sfc, userOpts=dict()):self.sf = sfc@@ -129,7 +131,7 @@ class sfp_dnscommonsrv(SpiderFootPlugin):try:answers = res.query(name, 'SRV')- except BaseException as e:+ except BaseException:answers = []for a in answers:diff --git a/modules/sfp_dnsgrep.py b/modules/sfp_dnsgrep.pyindex 144d2d7a..88403cf1 100644--- a/modules/sfp_dnsgrep.py+++ b/modules/sfp_dnsgrep.py@@ -16,7 +16,7 @@import jsonimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_dnsgrep(SpiderFootPlugin):"""DNSGrep:Footprint,Investigate,Passive:Passive DNS::Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API."""@@ -24,9 +24,9 @@ class sfp_dnsgrep(SpiderFootPlugin):meta = {'name': "DNSGrep",'summary': "Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "https://opendata.rapid7.com/",'model': "FREE_AUTH_UNLIMITED",@@ -99,7 +99,7 @@ class sfp_dnsgrep(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from DNSGrep.", False)+ self.sf.error(f"Error processing JSON response from DNSGrep: {e}", False)return Nonereturn datadiff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.pyindex 928db313..88b2807c 100644--- a/modules/sfp_dnsneighbor.py+++ b/modules/sfp_dnsneighbor.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_dnsneighbor(SpiderFootPlugin):"""DNS Look-aside:Footprint,Investigate:DNS::Attempt to reverse-resolve the IP addresses next to your target to see if they are related."""@@ -20,9 +20,9 @@ class sfp_dnsneighbor(SpiderFootPlugin):meta = {'name': "DNS Look-aside",'summary': "Attempt to reverse-resolve the IP addresses next to your target to see if they are related.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["DNS"]}# Default options@@ -79,15 +79,15 @@ class sfp_dnsneighbor(SpiderFootPlugin):try:ip = IPAddress(eventData)- except BaseException as e:- self.sf.error("Invalid IP address received: " + eventData, False)+ except BaseException:+ self.sf.error(f"Invalid IP address received: {eventData}", False)return Nonetry:minip = IPAddress(int(ip) - self.opts['lookasidecount'])maxip = IPAddress(int(ip) + self.opts['lookasidecount'])- except BaseException as e:- self.sf.error("Received an invalid IP address: " + eventData, False)+ except BaseException:+ self.sf.error(f"Received an invalid IP address: {eventData}", False)return Noneself.sf.debug("Lookaside max: " + str(maxip) + ", min: " + str(minip))diff --git a/modules/sfp_dnsraw.py b/modules/sfp_dnsraw.pyindex b63795a6..2ea72bc7 100644--- a/modules/sfp_dnsraw.py+++ b/modules/sfp_dnsraw.py@@ -16,7 +16,7 @@ import reimport dns.resolverimport dns.queryimport dns.rdatatype-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_dnsraw(SpiderFootPlugin):"""DNS Raw Records:Footprint,Investigate,Passive:DNS::Retrieves raw DNS records such as MX, TXT and others."""@@ -24,9 +24,9 @@ class sfp_dnsraw(SpiderFootPlugin):meta = {'name': "DNS Raw Records",'summary': "Retrieves raw DNS records such as MX, TXT and others.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["DNS"]}# Default optionsdiff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.pyindex ec4f03e0..76528d1b 100644--- a/modules/sfp_dnszonexfer.py+++ b/modules/sfp_dnszonexfer.py@@ -14,7 +14,7 @@ import reimport dns.queryimport dns.zone-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_dnszonexfer(SpiderFootPlugin):"""DNS Zone Transfer:Footprint,Investigate:DNS::Attempts to perform a full DNS zone transfer."""@@ -22,9 +22,9 @@ class sfp_dnszonexfer(SpiderFootPlugin):meta = {'name': "DNS Zone Transfer",'summary': "Attempts to perform a full DNS zone transfer.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["DNS"]}# Default options@@ -61,7 +61,6 @@ class sfp_dnszonexfer(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataeventDataHash = self.sf.hashstring(eventData)- addrs = NoneparentEvent = eventself.sf.debug(f"Received event, {eventName}, from {srcModuleName}")@@ -114,7 +113,7 @@ class sfp_dnszonexfer(SpiderFootPlugin):# Try and pull out individual recordsfor row in ret:- pat = re.compile("^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)+ pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)grps = re.findall(pat, row)if len(grps) > 0:for strdata in grps:diff --git a/modules/sfp_dronebl.py b/modules/sfp_dronebl.pyindex 2b3aa3ab..7298e755 100644--- a/modules/sfp_dronebl.py+++ b/modules/sfp_dronebl.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_dronebl(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_dronebl(SpiderFootPlugin):meta = {'name': "DroneBL",'summary': "Query the DroneBL database for open relays, open proxies, vulnerable servers, etc.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://dronebl.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -157,7 +157,6 @@ class sfp_dronebl(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataparentEvent = event- addrlist = list()self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.pyindex 84593b91..23ebc6fa 100644--- a/modules/sfp_duckduckgo.py+++ b/modules/sfp_duckduckgo.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_duckduckgo(SpiderFootPlugin):"""DuckDuckGo:Footprint,Investigate,Passive:Search Engines::Query DuckDuckGo's API for descriptive information about your target."""@@ -19,9 +19,9 @@ class sfp_duckduckgo(SpiderFootPlugin):meta = {'name': "DuckDuckGo",'summary': "Query DuckDuckGo's API for descriptive information about your target.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://duckduckgo.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -72,7 +72,6 @@ class sfp_duckduckgo(SpiderFootPlugin):def handleEvent(self, event):eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif self.opts['affiliatedomains'] and "AFFILIATE_" in eventName:@@ -97,6 +96,7 @@ class sfp_duckduckgo(SpiderFootPlugin):try:ret = json.loads(res['content'])except BaseException as e:+ self.sf.error(f"Error processing JSON response from DuckDuckGo: {e}", False)return Noneif ret['Heading'] == "":diff --git a/modules/sfp_email.py b/modules/sfp_email.pyindex 573d2872..a3d03ac4 100644--- a/modules/sfp_email.py+++ b/modules/sfp_email.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_email(SpiderFootPlugin):"""E-Mail Address Extractor:Footprint,Investigate,Passive:Content Analysis::Identify e-mail addresses in any obtained data."""@@ -19,8 +19,8 @@ class sfp_email(SpiderFootPlugin):meta = {'name': "E-Mail Address Extractor",'summary': "Identify e-mail addresses in any obtained data.",- 'useCases': [ "Passive", "Investigate", "Footprint" ],- 'categories': [ "Content Analysis" ]+ 'useCases': ["Passive", "Investigate", "Footprint"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_emailcrawlr.py b/modules/sfp_emailcrawlr.pyindex d584e4b2..e0db2d5a 100644--- a/modules/sfp_emailcrawlr.py+++ b/modules/sfp_emailcrawlr.py@@ -14,7 +14,7 @@import jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_emailcrawlr(SpiderFootPlugin):"""EmailCrawlr:Footprint,Investigate,Passive:Search Engines:apikey:Search EmailCrawlr for email addresses and phone numbers associated with a domain."""@@ -22,9 +22,9 @@ class sfp_emailcrawlr(SpiderFootPlugin):meta = {'name': "EmailCrawlr",'summary': "Search EmailCrawlr for email addresses and phone numbers associated with a domain.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://emailcrawlr.com/",'model': "FREE_AUTH_LIMITED",@@ -87,8 +87,8 @@ class sfp_emailcrawlr(SpiderFootPlugin):'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')}headers = {- 'Accept' : "application/json",- 'x-api-key' : self.opts['api_key']+ 'Accept': "application/json",+ 'x-api-key': self.opts['api_key']}res = self.sf.fetchUrl(@@ -135,7 +135,7 @@ class sfp_emailcrawlr(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn datadiff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.pyindex 404690d8..4c6c55da 100644--- a/modules/sfp_emailformat.py+++ b/modules/sfp_emailformat.py@@ -13,7 +13,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_emailformat(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_emailformat(SpiderFootPlugin):meta = {'name': "EmailFormat",'summary': "Look up e-mail addresses on email-format.com.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.email-format.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -94,7 +94,7 @@ class sfp_emailformat(SpiderFootPlugin):continue# Skip masked emails- if re.match("^[0-9a-f]{8}\.[0-9]{7}@", email):+ if re.match(r"^[0-9a-f]{8}\.[0-9]{7}@", email):self.sf.debug("Skipped address: " + email)continuediff --git a/modules/sfp_emailrep.py b/modules/sfp_emailrep.pyindex 594fb71c..6dad3d06 100644--- a/modules/sfp_emailrep.py+++ b/modules/sfp_emailrep.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_emailrep(SpiderFootPlugin):"""EmailRep:Footprint,Investigate,Passive:Search Engines:apikey:Search EmailRep.io for email address reputation."""@@ -20,9 +20,9 @@ class sfp_emailrep(SpiderFootPlugin):meta = {'name': "EmailRep",'summary': "Search EmailRep.io for email address reputation.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://emailrep.io/",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_emergingthreats.py b/modules/sfp_emergingthreats.pyindex bfe0fa2e..b07ec313 100644--- a/modules/sfp_emergingthreats.py+++ b/modules/sfp_emergingthreats.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_emergingthreats(SpiderFootPlugin):"""Emerging Threats:Investigate,Passive:Reputation Systems::Check if a netblock or IP is malicious according to emergingthreats.net."""@@ -19,9 +19,9 @@ class sfp_emergingthreats(SpiderFootPlugin):meta = {'name': "Emerging Threats",'summary': "Check if a netblock or IP is malicious according to emergingthreats.net.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://rules.emergingthreats.net/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_errors.py b/modules/sfp_errors.pyindex b2048102..792c3c0b 100644--- a/modules/sfp_errors.py+++ b/modules/sfp_errors.py@@ -12,17 +12,17 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEvent# Taken from Google Dorks on exploit-db.comregexps = dict({"PHP Error": ["PHP pase error", "PHP warning", "PHP error","unexpected T_VARIABLE", "warning: failed opening", "include_path="],"Generic Error": ["Internal Server Error", "Incorrect syntax"],- "Oracle Error": ["ORA-\d+", "TNS:.?no listen"],+ "Oracle Error": [r"ORA-\d+", "TNS:.?no listen"],"ASP Error": ["NET_SessionId"],- "MySQL Error": ["mysql_query\(", "mysql_connect\("],- "ODBC Error": ["\[ODBC SQL"]+ "MySQL Error": [r"mysql_query\(", r"mysql_connect\("],+ "ODBC Error": [r"\[ODBC SQL"]})@@ -32,9 +32,9 @@ class sfp_errors(SpiderFootPlugin):meta = {'name': "Error String Extractor",'summary': "Identify common error messages in content like SQL errors, etc.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_ethereum.py b/modules/sfp_ethereum.pyindex 4d712cde..99c1b223 100644--- a/modules/sfp_ethereum.py+++ b/modules/sfp_ethereum.py@@ -13,7 +13,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_ethereum(SpiderFootPlugin):"""Ethereum Address Extractor:Footprint,Investigate,Passive:Content Analysis::Identify ethereum addresses in scraped webpages."""@@ -21,9 +21,9 @@ class sfp_ethereum(SpiderFootPlugin):meta = {'name': "Ethereum Address Extractor",'summary': "Identify ethereum addresses in scraped webpages.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_filemeta.py b/modules/sfp_filemeta.pyindex df4cc24e..19040779 100644--- a/modules/sfp_filemeta.py+++ b/modules/sfp_filemeta.py@@ -17,7 +17,7 @@ import docximport pptximport exifreadimport io-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_filemeta(SpiderFootPlugin):@@ -26,9 +26,9 @@ class sfp_filemeta(SpiderFootPlugin):meta = {'name': "File Metadata Extractor",'summary': "Extracts meta data from documents and images.",- 'flags': [ "" ],- 'useCases': [ "Footprint" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.pyindex a3f676cc..26303d5c 100644--- a/modules/sfp_fortinet.py+++ b/modules/sfp_fortinet.py@@ -12,12 +12,12 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'Fortiguard Threat Lookup': {'id': '_fortiguard',- 'checks': [ 'ip' ],+ 'checks': ['ip'],'url': 'https://fortiguard.com/search?q={0}&engine=8','badregex': ['.*Your signature is blacklisted.*'],'goodregex': ['.*Your signature is not blacklisted.*']@@ -31,9 +31,9 @@ class sfp_fortinet(SpiderFootPlugin):meta = {'name': "Fortiguard.com",'summary': "Check if an IP is malicious according to Fortiguard.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://fortiguard.com/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_fraudguard.py b/modules/sfp_fraudguard.pyindex 92c02a5a..924b3a92 100644--- a/modules/sfp_fraudguard.py+++ b/modules/sfp_fraudguard.py@@ -15,7 +15,7 @@ import base64from datetime import datetimeimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_fraudguard(SpiderFootPlugin):"""Fraudguard:Investigate,Passive:Reputation Systems:apikey:Obtain threat information from Fraudguard.io"""@@ -23,9 +23,9 @@ class sfp_fraudguard(SpiderFootPlugin):meta = {'name': "Fraudguard",'summary': "Obtain threat information from Fraudguard.io",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://fraudguard.io/",'model': "FREE_AUTH_LIMITED",@@ -89,7 +89,7 @@ class sfp_fraudguard(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return [ "GEOINFO", "MALICIOUS_IPADDR", "MALICIOUS_NETBLOCK" ]+ return ["GEOINFO", "MALICIOUS_IPADDR", "MALICIOUS_NETBLOCK"]def query(self, qry):fraudguard_url = "https://api.fraudguard.io/ip/" + qry@@ -104,10 +104,10 @@ class sfp_fraudguard(SpiderFootPlugin):'Authorization': "Basic " + token.decode('utf-8')}- res = self.sf.fetchUrl(fraudguard_url , timeout=self.opts['_fetchtimeout'],+ res = self.sf.fetchUrl(fraudguard_url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot", headers=headers)- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("Fraudguard.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)self.errorState = Truereturn None@@ -119,7 +119,7 @@ class sfp_fraudguard(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from Fraudguard.io.", False)+ self.sf.error(f"Error processing JSON response from Fraudguard.io: {e}", False)return None#print str(info)diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.pyindex 1ab075c8..6735187d 100644--- a/modules/sfp_fringeproject.py+++ b/modules/sfp_fringeproject.py@@ -15,7 +15,7 @@ import reimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_fringeproject(SpiderFootPlugin):"""Fringe Project:Investigate,Footprint,Passive:Search Engines::Obtain network information from Fringe Project API."""@@ -23,9 +23,9 @@ class sfp_fringeproject(SpiderFootPlugin):meta = {'name': "Fringe Project",'summary': "Obtain network information from Fringe Project API.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Footprint", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Footprint", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://fringeproject.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -160,7 +160,7 @@ class sfp_fringeproject(SpiderFootPlugin):for tag in tags:try:port = re.findall(r'^port:([0-9]+)', tag)- except BaseException as e:+ except BaseException:self.sf.debug("Didn't get sane data from FringeProject.")continuediff --git a/modules/sfp_fsecure_riddler.py b/modules/sfp_fsecure_riddler.pyindex c0d3c520..96e87d46 100644--- a/modules/sfp_fsecure_riddler.py+++ b/modules/sfp_fsecure_riddler.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_fsecure_riddler(SpiderFootPlugin):"""F-Secure Riddler.io:Investigate,Footprint,Passive:Search Engines:apikey:Obtain network information from F-Secure Riddler.io API."""@@ -20,9 +20,9 @@ class sfp_fsecure_riddler(SpiderFootPlugin):meta = {'name': "F-Secure Riddler.io",'summary': "Obtain network information from F-Secure Riddler.io API.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Footprint", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Footprint", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://riddler.io/",'model': "PRIVATE_ONLY",@@ -139,7 +139,7 @@ class sfp_fsecure_riddler(SpiderFootPlugin):time.sleep(1)- if res['code'] in [ "400", "401", "402", "403" ]:+ if res['code'] in ["400", "401", "402", "403"]:self.sf.error('Unexpected HTTP response code: ' + res['code'], False)self.errorState = Truereturn Nonediff --git a/modules/sfp_fullcontact.py b/modules/sfp_fullcontact.pyindex a3dc12b8..b5cb2def 100644--- a/modules/sfp_fullcontact.py+++ b/modules/sfp_fullcontact.py@@ -12,7 +12,7 @@import jsonimport timefrom datetime import datetime-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_fullcontact(SpiderFootPlugin):"""FullContact:Footprint,Investigate,Passive:Search Engines:apikey:Gather domain and e-mail information from fullcontact.com."""@@ -20,9 +20,9 @@ class sfp_fullcontact(SpiderFootPlugin):meta = {'name': "FullContact",'summary': "Gather domain and e-mail information from fullcontact.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.fullcontact.com",'model': "FREE_AUTH_LIMITED",@@ -83,12 +83,12 @@ class sfp_fullcontact(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "DOMAIN_NAME", "EMAILADDR" ]+ return ["DOMAIN_NAME", "EMAILADDR"]# What events this module producesdef producedEvents(self):- return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",- "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS" ]+ return ["EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",+ "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS"]def query(self, url, data, failcount=0):header = "Bearer " + self.opts['api_key']@@ -98,7 +98,7 @@ class sfp_fullcontact(SpiderFootPlugin):useragent="SpiderFoot", postData=json.dumps(data),headers={"Authorization": header})- if res['code'] in [ "401", "400" ]:+ if res['code'] in ["401", "400"]:self.sf.error("API key rejected by fullcontact.com", False)self.errorState = Truereturn Nonediff --git a/modules/sfp_github.py b/modules/sfp_github.pyindex 396235bd..cd870a93 100644--- a/modules/sfp_github.py+++ b/modules/sfp_github.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_github(SpiderFootPlugin):"""Github:Footprint,Passive:Social Media::Identify associated public code repositories on Github."""@@ -20,9 +20,9 @@ class sfp_github(SpiderFootPlugin):meta = {'name': "Github",'summary': "Identify associated public code repositories on Github.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://github.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -38,12 +38,12 @@ class sfp_github(SpiderFootPlugin):# Default optionsopts = {- 'namesonly': True+ 'namesonly': True}# Option descriptionsoptdescs = {- 'namesonly': "Match repositories by name only, not by their descriptions. Helps reduce false positives."+ 'namesonly': "Match repositories by name only, not by their descriptions. Helps reduce false positives."}def setup(self, sfc, userOpts=dict()):@@ -85,7 +85,6 @@ class sfp_github(SpiderFootPlugin):def handleEvent(self, event):eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif eventData in self.results:@@ -111,7 +110,7 @@ class sfp_github(SpiderFootPlugin):try:bits = url.split("/")name = bits[len(bits)-1]- except BaseException as e:+ except BaseException:self.sf.debug("Couldn't get a username out of " + url)return None@@ -123,7 +122,7 @@ class sfp_github(SpiderFootPlugin):try:json_data = json.loads(res['content'])except BaseException as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Noneif not json_data.get('login'):@@ -161,6 +160,7 @@ class sfp_github(SpiderFootPlugin):self.sf.debug("Looking at " + name)failed = False+# Get all the repositories based on direct matches with the# name identifiedurl = "https://api.github.com/search/repositories?q=" + name@@ -174,11 +174,11 @@ class sfp_github(SpiderFootPlugin):try:ret = json.loads(res['content'])except BaseException as e:+ self.sf.debug(f"Error processing JSON response from GitHub: {e}", False)ret = None- if ret == None:- self.sf.error("Unable to process empty response from Github for: " + \- name, False)+ if ret is None:+ self.sf.error(f"Unable to process empty response from Github for: {name}", False)failed = Trueif not failed:@@ -211,12 +211,10 @@ class sfp_github(SpiderFootPlugin):try:ret = json.loads(res['content'])if ret == None:- self.sf.error("Unable to process empty response from Github for: " + \- name, False)+ self.sf.error(f"Unable to process empty response from Github for: {name}", False)failed = True- except BaseException as e:- self.sf.error("Unable to process invalid response from Github for: " + \- name, False)+ except BaseException:+ self.sf.error(f"Unable to process invalid response from Github for: {name}", False)failed = Trueif not failed:@@ -242,7 +240,7 @@ class sfp_github(SpiderFootPlugin):try:repret = json.loads(res['content'])except BaseException as e:- self.sf.error("Invalid JSON returned from Github.", False)+ self.sf.error(f"Invalid JSON returned from Github: {e}", False)continueif repret == None:diff --git a/modules/sfp_googlemaps.py b/modules/sfp_googlemaps.pyindex d9316b6a..fa68e6a7 100644--- a/modules/sfp_googlemaps.py+++ b/modules/sfp_googlemaps.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_googlemaps(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_googlemaps(SpiderFootPlugin):meta = {'name': "Google Maps",'summary': "Identifies potential physical addresses and latitude/longitude coordinates.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Real World" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Real World"],'dataSource': {'website': "https://cloud.google.com/maps-platform/",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_googleobjectstorage.py b/modules/sfp_googleobjectstorage.pyindex 1e14fcf9..6e00f831 100644--- a/modules/sfp_googleobjectstorage.py+++ b/modules/sfp_googleobjectstorage.py@@ -13,7 +13,7 @@import threadingimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_googleobjectstorage(SpiderFootPlugin):"""Google Object Storage Finder:Footprint,Passive:Crawling and Scanning::Search for potential Google Object Storage buckets associated with the target and attempt to list their contents."""@@ -21,9 +21,9 @@ class sfp_googleobjectstorage(SpiderFootPlugin):meta = {'name': "Google Object Storage Finder",'summary': "Search for potential Google Object Storage buckets associated with the target and attempt to list their contents.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Crawling and Scanning" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Crawling and Scanning"],'dataSource': {'website': "https://cloud.google.com/storage",'model': "FREE_NOAUTH_UNLIMITED"@@ -68,7 +68,7 @@ class sfp_googleobjectstorage(SpiderFootPlugin):def checkSite(self, url):res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)- if res['code'] not in [ "301", "302", "200" ] and \+ if res['code'] not in ["301", "302", "200"] and \(res['content'] is None or "NoSuchBucket" in res['content']):self.sf.debug("Not a valid bucket: " + url)else:@@ -80,7 +80,6 @@ class sfp_googleobjectstorage(SpiderFootPlugin):self.gosresults[url] = 0def threadSites(self, siteList):- ret = list()self.gosresults = dict()running = Truei = 0@@ -154,7 +153,7 @@ class sfp_googleobjectstorage(SpiderFootPlugin):self.notifyListeners(evt)return None- targets = [ eventData.replace('.', '') ]+ targets = [eventData.replace('.', '')]kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])if kw:targets.append(kw)diff --git a/modules/sfp_googlesafebrowsing.py b/modules/sfp_googlesafebrowsing.pyindex 99acca38..df7567e3 100644--- a/modules/sfp_googlesafebrowsing.py+++ b/modules/sfp_googlesafebrowsing.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_googlesafebrowsing(SpiderFootPlugin):@@ -161,7 +161,7 @@ class sfp_googlesafebrowsing(SpiderFootPlugin):return Noneexcept Exception as e:- self.sf.error("Error processing JSON response from SHODAN.", False)+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)return Nonereturn infodiff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.pyindex 88e51d4d..d8d926ae 100644--- a/modules/sfp_googlesearch.py+++ b/modules/sfp_googlesearch.py@@ -10,7 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_googlesearch(SpiderFootPlugin):@@ -19,9 +19,9 @@ class sfp_googlesearch(SpiderFootPlugin):meta = {'name': "Google",'summary': "Obtain information from the Google Custom Search API to identify sub-domains and links.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://developers.google.com/custom-search",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.pyindex 54b5f8ee..0c7b930e 100644--- a/modules/sfp_gravatar.py+++ b/modules/sfp_gravatar.py@@ -14,7 +14,7 @@import jsonimport hashlibimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_gravatar(SpiderFootPlugin):"""Gravatar:Footprint,Investigate,Passive:Social Media::Retrieve user information from Gravatar API."""@@ -22,9 +22,9 @@ class sfp_gravatar(SpiderFootPlugin):meta = {'name': "Gravatar",'summary': "Retrieve user information from Gravatar API.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://secure.gravatar.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -74,7 +74,7 @@ class sfp_gravatar(SpiderFootPlugin):# https://secure.gravatar.com/site/implement/# https://secure.gravatar.com/site/implement/profiles/def query(self, qry):- email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest() # nosec+ email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest() # nosecoutput = 'json'res = self.sf.fetchUrl("https://secure.gravatar.com/" + email_hash + '.' + output,diff --git a/modules/sfp_greensnow.py b/modules/sfp_greensnow.pyindex b75d4025..a9def062 100644--- a/modules/sfp_greensnow.py+++ b/modules/sfp_greensnow.py@@ -12,7 +12,7 @@from netaddr import IPAddress, IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_greensnow(SpiderFootPlugin):"""Greensnow:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is malicious according to greensnow.co."""@@ -20,9 +20,9 @@ class sfp_greensnow(SpiderFootPlugin):meta = {'name': "Greensnow",'summary': "Check if a netblock or IP address is malicious according to greensnow.co.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://greensnow.co/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_grep_app.py b/modules/sfp_grep_app.pyindex e7c7c737..660c3ef5 100644--- a/modules/sfp_grep_app.py+++ b/modules/sfp_grep_app.py@@ -15,7 +15,7 @@ import jsonimport mathimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_grep_app(SpiderFootPlugin):"""grep.app:Footprint,Investigate,Passive:Search Engines::Search grep.app API for links and emails related to the specified domain."""@@ -23,9 +23,9 @@ class sfp_grep_app(SpiderFootPlugin):meta = {'name': "grep.app",'summary': "Search grep.app API for links and emails related to the specified domain.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://grep.app/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_greynoise.py b/modules/sfp_greynoise.pyindex b65ff455..eea93375 100644--- a/modules/sfp_greynoise.py+++ b/modules/sfp_greynoise.py@@ -14,7 +14,7 @@ import jsonfrom datetime import datetimeimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_greynoise(SpiderFootPlugin):"""Greynoise:Investigate,Passive:Reputation Systems:apikey:Obtain information from Greynoise.io's Enterprise API."""@@ -22,9 +22,9 @@ class sfp_greynoise(SpiderFootPlugin):meta = {'name': "Greynoise",'summary': "Obtain information from Greynoise.io's Enterprise API.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://greynoise.io/",'model': "FREE_AUTH_LIMITED",@@ -97,17 +97,15 @@ class sfp_greynoise(SpiderFootPlugin):return ["MALICIOUS_IPADDR", "MALICIOUS_ASN", "MALICIOUS_SUBNET","MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK","COMPANY_NAME", "GEOINFO", "BGP_AS_MEMBER", "OPERATING_SYSTEM",- "RAW_RIR_DATA" ]+ "RAW_RIR_DATA"]def queryIP(self, qry):- ret = None-- header = { "key": self.opts['api_key'] }+ header = {"key": self.opts['api_key']}url = "https://enterprise.api.greynoise.io/v2/experimental/gnql?query=" + qryres = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot", headers=header)- if res['code'] not in [ "200" ]:+ if res['code'] not in ["200"]:self.sf.error("Greynoise API key seems to have been rejected or you have exceeded usage limits.", False)self.errorState = Truereturn None@@ -115,7 +113,7 @@ class sfp_greynoise(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from Greynoise.", False)+ self.sf.error(f"Error processing JSON response from Greynoise: {e}", False)return Nonereturn infodiff --git a/modules/sfp_h1nobbdde.py b/modules/sfp_h1nobbdde.pyindex 06c9c7d1..3a7ba092 100644--- a/modules/sfp_h1nobbdde.py+++ b/modules/sfp_h1nobbdde.py@@ -11,7 +11,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_h1nobbdde(SpiderFootPlugin):"""HackerOne (Unofficial):Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed."""@@ -19,9 +19,9 @@ class sfp_h1nobbdde(SpiderFootPlugin):meta = {'name': "HackerOne (Unofficial)",'summary': "Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "http://www.nobbd.de/",'model': "FREE_NOAUTH_UNLIMITED",@@ -68,7 +68,6 @@ class sfp_h1nobbdde(SpiderFootPlugin):# Query h1.nobbd.dedef queryOBB(self, qry):ret = list()- base = "http://www.h1.nobbd.de"url = "http://h1.nobbd.de/search.php?q=" + qryres = self.sf.fetchUrl(url, timeout=30, useragent=self.opts['_useragent'])diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.pyindex 00ee6571..961d3c21 100644--- a/modules/sfp_hackertarget.py+++ b/modules/sfp_hackertarget.py@@ -16,7 +16,7 @@ import reimport jsonfrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_hackertarget(SpiderFootPlugin):@@ -25,9 +25,9 @@ class sfp_hackertarget(SpiderFootPlugin):meta = {'name': "HackerTarget",'summary': "Search HackerTarget.com for hosts sharing the same IP.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "https://hackertarget.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -247,7 +247,7 @@ class sfp_hackertarget(SpiderFootPlugin):# Try and pull out individual recordsfor row in records:- pat = re.compile("^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)+ pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL)grps = re.findall(pat, row)if len(grps) == 0:diff --git a/modules/sfp_hashes.py b/modules/sfp_hashes.pyindex 7d68e2f8..2817e9da 100644--- a/modules/sfp_hashes.py+++ b/modules/sfp_hashes.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_hashes(SpiderFootPlugin):"""Hash Extractor:Footprint,Investigate,Passive:Content Analysis::Identify MD5 and SHA hashes in web content, files and more."""@@ -19,9 +19,9 @@ class sfp_hashes(SpiderFootPlugin):meta = {'name': "Hash Extractor",'summary': "Identify MD5 and SHA hashes in web content, files and more.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default options@@ -60,7 +60,6 @@ class sfp_hashes(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")hashes = self.sf.parseHashes(eventData)- myres = list()for hashtup in hashes:hashalgo, hashval = hashtupdiff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.pyindex 79d65bb0..e8bce9fa 100644--- a/modules/sfp_haveibeenpwned.py+++ b/modules/sfp_haveibeenpwned.py@@ -11,7 +11,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_haveibeenpwned(SpiderFootPlugin):"""HaveIBeenPwned:Footprint,Investigate,Passive:Leaks, Dumps and Breaches:apikey:Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches."""@@ -19,9 +19,9 @@ class sfp_haveibeenpwned(SpiderFootPlugin):meta = {'name': "HaveIBeenPwned",'summary': "Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://haveibeenpwned.com/",'model': "COMMERCIAL_ONLY",@@ -74,12 +74,12 @@ class sfp_haveibeenpwned(SpiderFootPlugin):def query(self, qry):ret = Noneif self.opts['api_key']:- v = "3"+ version = "3"else:- v = "2"+ version = "2"- url = "https://haveibeenpwned.com/api/v" + v + "/breachedaccount/" + qry- hdrs = { "Accept": "application/vnd.haveibeenpwned.v" + v + "+json" }+ url = f"https://haveibeenpwned.com/api/v{version}/breachedaccount/{qry}"+ hdrs = {"Accept": f"application/vnd.haveibeenpwned.v{version}+json"}retry = 0if self.opts['api_key']:@@ -110,7 +110,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):try:ret = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from HaveIBeenPwned?: " + str(e), False)+ self.sf.error(f"Error processing JSON response from HaveIBeenPwned?: {e}", False)return Nonereturn ret@@ -144,7 +144,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):else:site = n["Name"]except BaseException as e:- self.sf.debug("Unable to parse result from HaveIBeenPwned?")+ self.sf.debug(f"Unable to parse result from HaveIBeenPwned?: {e}")continueevt = eventName + "_COMPROMISED"diff --git a/modules/sfp_honeypot.py b/modules/sfp_honeypot.pyindex 93b1d50c..0a400471 100644--- a/modules/sfp_honeypot.py+++ b/modules/sfp_honeypot.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_honeypot(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_honeypot(SpiderFootPlugin):meta = {'name': "Honeypot Checker",'summary': "Query the projecthoneypot.org database for entries.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.projecthoneypot.org/",'model': "FREE_AUTH_UNLIMITED",@@ -177,7 +177,6 @@ class sfp_honeypot(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataparentEvent = event- addrlist = list()if self.errorState:return Nonediff --git a/modules/sfp_hosting.py b/modules/sfp_hosting.pyindex 3a3ac170..0840d9e8 100644--- a/modules/sfp_hosting.py+++ b/modules/sfp_hosting.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_hosting(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_hosting(SpiderFootPlugin):meta = {'name': "Hosting Provider Identifier",'summary': "Find out if any IP addresses identified fall within known 3rd party hosting ranges, e.g. Amazon, Azure, etc.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default options@@ -73,7 +73,7 @@ class sfp_hosting(SpiderFootPlugin):continuetry:[start, end, title, url] = line.split(",")- except BaseException as e:+ except BaseException:continuetry:diff --git a/modules/sfp_hunter.py b/modules/sfp_hunter.pyindex b307e7e1..a94bea1c 100644--- a/modules/sfp_hunter.py+++ b/modules/sfp_hunter.py@@ -11,7 +11,7 @@import jsonimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_hunter(SpiderFootPlugin):"""Hunter.io:Footprint,Investigate,Passive:Search Engines:apikey:Check for e-mail addresses and names on hunter.io."""@@ -19,9 +19,9 @@ class sfp_hunter(SpiderFootPlugin):meta = {'name': "Hunter.io",'summary': "Check for e-mail addresses and names on hunter.io.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://hunter.io/",'model': "FREE_AUTH_LIMITED",@@ -73,11 +73,11 @@ class sfp_hunter(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "DOMAIN_NAME", "INTERNET_NAME" ]+ return ["DOMAIN_NAME", "INTERNET_NAME"]# What events this module producesdef producedEvents(self):- return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA" ]+ return ["EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA"]def query(self, qry, offset=0, limit=10):params = {@@ -91,8 +91,7 @@ class sfp_hunter(SpiderFootPlugin):url = "https://api.hunter.io/v2/domain-search?%s" % urllib.parse.urlencode(params)- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent="SpiderFoot")+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")if res['code'] == "404":return Nonediff --git a/modules/sfp_hybrid_analysis.py b/modules/sfp_hybrid_analysis.pyindex cb9e6d04..4e7f9ca2 100644--- a/modules/sfp_hybrid_analysis.py+++ b/modules/sfp_hybrid_analysis.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_hybrid_analysis(SpiderFootPlugin):"""Hybrid Analysis:Footprint,Investigate,Passive:Reputation Systems:apikey:Search Hybrid Analysis for domains and URLs related to the target."""@@ -20,9 +20,9 @@ class sfp_hybrid_analysis(SpiderFootPlugin):meta = {'name': "Hybrid Analysis",'summary': "Search Hybrid Analysis for domains and URLs related to the target.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "www.hybrid-analysis.com",'model': "FREE_AUTH_UNLIMITED",@@ -144,7 +144,7 @@ class sfp_hybrid_analysis(SpiderFootPlugin):"hash": qry.encode('raw_unicode_escape').decode("ascii", errors='replace')}headers = {- "Accept" : "application/json",+ "Accept": "application/json",'api-key': self.opts['api_key']}res = self.sf.fetchUrl(@@ -187,7 +187,7 @@ class sfp_hybrid_analysis(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn datadiff --git a/modules/sfp_iban.py b/modules/sfp_iban.pyindex a59c1e6b..3cd37297 100644--- a/modules/sfp_iban.py+++ b/modules/sfp_iban.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_iban(SpiderFootPlugin):"""IBAN Number Extractor:Footprint,Investigate,Passive:Content Analysis::Identify IBAN Numbers in any data"""@@ -19,9 +19,9 @@ class sfp_iban(SpiderFootPlugin):meta = {'name': "IBAN Number Extractor",'summary': "Identify IBAN Numbers in any data",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default options.diff --git a/modules/sfp_iknowwhatyoudownload.py b/modules/sfp_iknowwhatyoudownload.pyindex d41066f7..78d84fe9 100644--- a/modules/sfp_iknowwhatyoudownload.py+++ b/modules/sfp_iknowwhatyoudownload.py@@ -10,7 +10,7 @@#-------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_iknowwhatyoudownload(SpiderFootPlugin):"""Iknowwhatyoudownload.com:Footprint,Investigate,Passive:Secondary Networks:apikey:Check iknowwhatyoudownload.com for IP addresses that have been using BitTorrent."""@@ -18,9 +18,9 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):meta = {'name': "Iknowwhatyoudownload.com",'summary': "Check iknowwhatyoudownload.com for IP addresses that have been using BitTorrent.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Secondary Networks" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Secondary Networks"],'dataSource': {'website': "https://iknowwhatyoudownload.com/en/peer/",'model': "FREE_AUTH_LIMITED",@@ -91,8 +91,7 @@ class sfp_iknowwhatyoudownload(SpiderFootPlugin):url = base + qry + "&days=" + str(self.opts['daysback'])url += "&key=" + self.opts['api_key']- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent="SpiderFoot")+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")if res['code'] in ["403", "500"]:self.sf.info("Unable to fetch data from iknowwhatyoudownload.com right now.")diff --git a/modules/sfp_instagram.py b/modules/sfp_instagram.pyindex d159208a..b52a4c7c 100644--- a/modules/sfp_instagram.py+++ b/modules/sfp_instagram.py@@ -12,7 +12,7 @@import jsonimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_instagram(SpiderFootPlugin):"""Instagram:Footprint,Investigate,Passive:Social Media::Gather information from Instagram profiles."""@@ -20,9 +20,9 @@ class sfp_instagram(SpiderFootPlugin):meta = {'name': "Instagram",'summary': "Gather information from Instagram profiles.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://www.instagram.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -55,11 +55,11 @@ class sfp_instagram(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ 'SOCIAL_MEDIA' ]+ return ['SOCIAL_MEDIA']# What events this module producesdef producedEvents(self):- return [ 'RAW_RIR_DATA' ]+ return ['RAW_RIR_DATA']# Extract profile JSON from HTMLdef extractJson(self, html):diff --git a/modules/sfp_intelx.py b/modules/sfp_intelx.pyindex bb3264d8..afc0764a 100644--- a/modules/sfp_intelx.py+++ b/modules/sfp_intelx.py@@ -14,7 +14,7 @@import jsonimport timeimport datetime-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_intelx(SpiderFootPlugin):@@ -23,9 +23,9 @@ class sfp_intelx(SpiderFootPlugin):meta = {'name': "IntelligenceX",'summary': "Obtain information from IntelligenceX about identified IP addresses, domains, e-mail addresses and phone numbers.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://intelx.io/",'model': "FREE_AUTH_LIMITED",@@ -252,15 +252,15 @@ class sfp_intelx(SpiderFootPlugin):try:val = rec['selectorvalueh']evt = None- if rec['selectortype'] == 1: #Email+ if rec['selectortype'] == 1: # Emailevt = "EMAILADDR"if val.split("@")[0] in self.opts['_genericusers'].split(","):evt = "EMAILADDR_GENERIC"- if rec['selectortype'] == 2: #Domain+ if rec['selectortype'] == 2: # Domainevt = "INTERNET_NAME"if val == eventData:continue- if rec['selectortype'] == 3: #URL+ if rec['selectortype'] == 3: # URLevt = "LINKED_URL_INTERNAL"if not val or not evt:diff --git a/modules/sfp_intfiles.py b/modules/sfp_intfiles.pyindex b7c630dd..88781812 100644--- a/modules/sfp_intfiles.py+++ b/modules/sfp_intfiles.py@@ -10,7 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_intfiles(SpiderFootPlugin):@@ -19,9 +19,9 @@ class sfp_intfiles(SpiderFootPlugin):meta = {'name': "Interesting File Finder",'summary': "Identifies potential files of interest, e.g. office documents, zip files.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Crawling and Scanning" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Crawling and Scanning"]}# Default optionsdiff --git a/modules/sfp_ipinfo.py b/modules/sfp_ipinfo.pyindex 8213b776..c5230a5f 100644--- a/modules/sfp_ipinfo.py+++ b/modules/sfp_ipinfo.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_ipinfo(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_ipinfo(SpiderFootPlugin):meta = {'name': "IPInfo.io",'summary': "Identifies the physical location of IP addresses identified using ipinfo.io.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Real World" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Real World"],'dataSource': {'website': "https://ipinfo.io",'model': "FREE_AUTH_LIMITED",@@ -95,7 +95,7 @@ class sfp_ipinfo(SpiderFootPlugin):try:result = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn resultdiff --git a/modules/sfp_ipstack.py b/modules/sfp_ipstack.pyindex 979507bc..e1079dc2 100644--- a/modules/sfp_ipstack.py+++ b/modules/sfp_ipstack.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_ipstack(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_ipstack(SpiderFootPlugin):meta = {'name': "ipstack",'summary': "Identifies the physical location of IP addresses identified using ipstack.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Real World" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Real World"],'dataSource': {'website': "https://ipstack.com/",'model': "FREE_AUTH_LIMITED",@@ -108,7 +108,7 @@ class sfp_ipstack(SpiderFootPlugin):self.errorState = Truereturn Noneexcept Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Noneif hostip.get('country_name', None) != None:diff --git a/modules/sfp_isc.py b/modules/sfp_isc.pyindex 71b5a386..42c96d8b 100644--- a/modules/sfp_isc.py+++ b/modules/sfp_isc.py@@ -12,14 +12,14 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'Internet Storm Center': {'id': '_isc','checks': ['ip'],'url': 'https://isc.sans.edu/api/ip/{0}',- 'badregex': ['.*<attacks>\d+</attacks>.*'],+ 'badregex': [r'.*<attacks>\d+</attacks>.*'],'goodregex': []}}@@ -31,9 +31,9 @@ class sfp_isc(SpiderFootPlugin):meta = {'name': "Internet Storm Center",'summary': "Check if an IP is malicious according to SANS ISC.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://isc.sans.edu",'model': "FREE_NOAUTH_UNLIMITED",@@ -87,7 +87,7 @@ class sfp_isc(SpiderFootPlugin):# What events is this module interested in for input# * = be notified about all events.def watchedEvents(self):- return ["IP_ADDRESS", "AFFILIATE_IPADDR" ]+ return ["IP_ADDRESS", "AFFILIATE_IPADDR"]# What events this module produces# This is to support the end user in selecting modules based on eventsdiff --git a/modules/sfp_jsonwhoiscom.py b/modules/sfp_jsonwhoiscom.pyindex 8a365acd..18cac5f5 100644--- a/modules/sfp_jsonwhoiscom.py+++ b/modules/sfp_jsonwhoiscom.py@@ -13,7 +13,7 @@import jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_jsonwhoiscom(SpiderFootPlugin):"""JsonWHOIS.com:Footprint,Investigate,Passive:Search Engines:apikey:Search JsonWHOIS.com for WHOIS records associated with a domain."""@@ -21,9 +21,9 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):meta = {'name': "JsonWHOIS.com",'summary': "Search JsonWHOIS.com for WHOIS records associated with a domain.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://jsonwhois.com",'model': "FREE_AUTH_LIMITED",@@ -86,8 +86,8 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')}headers = {- "Accept" : "application/json",- "Authorization" : "Token token=" + self.opts["api_key"]+ "Accept": "application/json",+ "Authorization": "Token token=" + self.opts["api_key"]}res = self.sf.fetchUrl(@@ -139,7 +139,7 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn datadiff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.pyindex 49cc5af0..f1798608 100644--- a/modules/sfp_junkfiles.py+++ b/modules/sfp_junkfiles.py@@ -10,7 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport randomclass sfp_junkfiles(SpiderFootPlugin):@@ -19,9 +19,9 @@ class sfp_junkfiles(SpiderFootPlugin):meta = {'name': "Junk File Finder",'summary': "Looks for old/temporary and other similar files.",- 'flags': [ "slow", "errorprone", "invasive" ],- 'useCases': [ "Footprint" ],- 'categories': [ "Crawling and Scanning" ]+ 'flags': ["slow", "errorprone", "invasive"],+ 'useCases': ["Footprint"],+ 'categories': ["Crawling and Scanning"]}# Default optionsdiff --git a/modules/sfp_keybase.py b/modules/sfp_keybase.pyindex 7a12b8b7..2677be95 100644--- a/modules/sfp_keybase.py+++ b/modules/sfp_keybase.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport urllib.request, urllib.parse, urllib.errorimport jsonimport re@@ -22,9 +22,9 @@ class sfp_keybase(SpiderFootPlugin):meta = {'name': "Keybase",'summary': "Obtain additional information about target username",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Public Registries" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Public Registries"],'dataSource': {'website': "https://keybase.io/",'model': "FREE_NOAUTH_UNLIMITED",@@ -62,9 +62,11 @@ class sfp_keybase(SpiderFootPlugin):return ["USERNAME", "LINKED_URL_EXTERNAL"]def producedEvents(self):- return ["RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME",+ return [+ "RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME","HUMAN_NAME", "GEOINFO", "BITCOIN_ADDRESS",- "PGP_KEY"]+ "PGP_KEY"+ ]def queryUsername(self, qry):@@ -73,7 +75,7 @@ class sfp_keybase(SpiderFootPlugin):}headers = {- 'Accept' : "application/json"+ 'Accept': "application/json"}res = self.sf.fetchUrl(@@ -129,7 +131,7 @@ class sfp_keybase(SpiderFootPlugin):# Extract username if a Keybase link is receivedif eventName == "LINKED_URL_EXTERNAL":- linkRegex = "^https?://keybase.io\/[A-Za-z0-9\-_\.]+"+ linkRegex = r"^https?://keybase.io\/[A-Za-z0-9\-_\.]+"links = re.findall(linkRegex, eventData)if len(links) == 0:@@ -229,9 +231,8 @@ class sfp_keybase(SpiderFootPlugin):evt = SpiderFootEvent("BITCOIN_ADDRESS", btcAddress, self.__name__, event)self.notifyListeners(evt)-# Extract PGP Keys- pgpRegex = "-----BEGIN\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----(.*?)-----END\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----"+ pgpRegex = r"-----BEGIN\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----(.*?)-----END\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----"pgpKeys = re.findall(pgpRegex, str(content))diff --git a/modules/sfp_leakix.py b/modules/sfp_leakix.pyindex d6627a86..11c17b58 100644--- a/modules/sfp_leakix.py+++ b/modules/sfp_leakix.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_leakix(SpiderFootPlugin):"""LeakIX:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Search LeakIX for host data leaks, open ports, software and geoip."""@@ -20,9 +20,9 @@ class sfp_leakix(SpiderFootPlugin):meta = {'name': "LeakIX",'summary': "Search LeakIX for host data leaks, open ports, software and geoip.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://leakix.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -73,7 +73,7 @@ class sfp_leakix(SpiderFootPlugin):# https://leakix.net/api-documentationdef queryHost(self, qry):headers = {- "Accept" : "application/json"+ "Accept": "application/json"}res = self.sf.fetchUrl('https://leakix.net/host/' + qry,@@ -110,7 +110,7 @@ class sfp_leakix(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn datadiff --git a/modules/sfp_malwaredomainlist.py b/modules/sfp_malwaredomainlist.pyindex 6213ec15..6b0eadeb 100644--- a/modules/sfp_malwaredomainlist.py+++ b/modules/sfp_malwaredomainlist.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'malwaredomainlist.com IP List': {@@ -25,7 +25,7 @@ malchecks = {'id': 'malwaredomainlistdomain','checks': ['domain'],'url': 'http://www.malwaredomainlist.com/hostslist/hosts.txt',- 'regex': '.*\s+{0}[\s$]'+ 'regex': r'.*\s+{0}[\s$]'}}@@ -35,9 +35,9 @@ class sfp_malwaredomainlist(SpiderFootPlugin):meta = {'name': "malwaredomainlist.com",'summary': "Check if a host/domain, IP or netblock is malicious according to malwaredomainlist.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.malwaredomainlist.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -135,8 +135,7 @@ class sfp_malwaredomainlist(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_malwaredomains.py b/modules/sfp_malwaredomains.pyindex 88816c8a..d3f4dd5c 100644--- a/modules/sfp_malwaredomains.py+++ b/modules/sfp_malwaredomains.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'malwaredomains.com Domains': {@@ -30,9 +30,9 @@ class sfp_malwaredomains(SpiderFootPlugin):meta = {'name': "malwaredomains.com",'summary': "Check if a host/domain is malicious according to malwaredomains.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.malwaredomains.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -122,8 +122,7 @@ class sfp_malwaredomains(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_malwarepatrol.py b/modules/sfp_malwarepatrol.pyindex 9422def4..27d85077 100644--- a/modules/sfp_malwarepatrol.py+++ b/modules/sfp_malwarepatrol.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_malwarepatrol(SpiderFootPlugin):@@ -20,9 +20,9 @@ class sfp_malwarepatrol(SpiderFootPlugin):meta = {'name': "MalwarePatrol",'summary': "Searches malwarepatrol.net's database of malicious URLs/IPs.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.malwarepatrol.net/",'model': "FREE_AUTH_LIMITED",@@ -132,7 +132,7 @@ class sfp_malwarepatrol(SpiderFootPlugin):evtType = 'MALICIOUS_AFFILIATE_IPADDR'if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',- 'AFFILIATE_INTERNET_NAME' ]:+ 'AFFILIATE_INTERNET_NAME']:if eventName == "INTERNET_NAME":evtType = "MALICIOUS_INTERNET_NAME"if eventName == 'AFFILIATE_INTERNET_NAME':diff --git a/modules/sfp_metadefender.py b/modules/sfp_metadefender.pyindex bbd66a7d..6524e056 100644--- a/modules/sfp_metadefender.py+++ b/modules/sfp_metadefender.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_metadefender(SpiderFootPlugin):"""MetaDefender:Footprint,Investigate,Passive:Reputation Systems:apikey:Search MetaDefender API for IP address and domain IP reputation."""@@ -20,9 +20,9 @@ class sfp_metadefender(SpiderFootPlugin):meta = {'name': "MetaDefender",'summary': "Search MetaDefender API for IP address and domain IP reputation.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ]+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Reputation Systems"]}# Default options@@ -114,7 +114,7 @@ class sfp_metadefender(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn datadiff --git a/modules/sfp_mnemonic.py b/modules/sfp_mnemonic.pyindex f99bf48b..dee2e7b8 100644--- a/modules/sfp_mnemonic.py+++ b/modules/sfp_mnemonic.py@@ -14,7 +14,7 @@import jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_mnemonic(SpiderFootPlugin):"""Mnemonic PassiveDNS:Footprint,Investigate,Passive:Passive DNS::Obtain Passive DNS information from PassiveDNS.mnemonic.no."""@@ -22,9 +22,9 @@ class sfp_mnemonic(SpiderFootPlugin):meta = {'name': "Mnemonic PassiveDNS",'summary': "Obtain Passive DNS information from PassiveDNS.mnemonic.no.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "https://www.mnemonic.no",'model': "FREE_NOAUTH_UNLIMITED",@@ -109,7 +109,7 @@ class sfp_mnemonic(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return None# Check the response is ok@@ -167,7 +167,7 @@ class sfp_mnemonic(SpiderFootPlugin):max_pages = int(self.opts['max_pages'])per_page = int(self.opts['per_page'])agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])- cohostcount = 0+ self.cohostcount = 0cohosts = list()while position < (per_page * max_pages):@@ -193,11 +193,11 @@ class sfp_mnemonic(SpiderFootPlugin):self.sf.debug("Record found too old, skipping.")continue- if eventName in [ 'IP_ADDRESS' ]:+ if eventName in ['IP_ADDRESS']:if r['rrtype'] == 'a':cohosts.append(r['query'])- if eventName in [ 'INTERNET_NAME', 'DOMAIN_NAME' ]:+ if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:# Ignore PTR recordsif r['rrtype'] == 'ptr':diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.pyindex 22904225..d70614c9 100644--- a/modules/sfp_multiproxy.py+++ b/modules/sfp_multiproxy.py@@ -13,7 +13,7 @@import refrom netaddr import IPAddress, IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'multiproxy.org Open Proxies': {@@ -31,9 +31,9 @@ class sfp_multiproxy(SpiderFootPlugin):meta = {'name': "multiproxy.org Open Proxies",'summary': "Check if an IP is an open proxy according to multiproxy.org' open proxy list.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Secondary Networks" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Secondary Networks"],'dataSource': {'website': "https://multiproxy.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -123,8 +123,7 @@ class sfp_multiproxy(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.pyindex 07a0a520..669a887b 100644--- a/modules/sfp_myspace.py+++ b/modules/sfp_myspace.py@@ -11,7 +11,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_myspace(SpiderFootPlugin):"""MySpace:Footprint,Investigate,Passive:Social Media::Gather username and location from MySpace.com profiles."""@@ -19,9 +19,9 @@ class sfp_myspace(SpiderFootPlugin):meta = {'name': "MySpace",'summary': "Gather username and location from MySpace.com profiles.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://myspace.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -58,11 +58,11 @@ class sfp_myspace(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "EMAILADDR", "SOCIAL_MEDIA" ]+ return ["EMAILADDR", "SOCIAL_MEDIA"]# What events this module producesdef producedEvents(self):- return [ "SOCIAL_MEDIA", "GEOINFO" ]+ return ["SOCIAL_MEDIA", "GEOINFO"]# Handle events sent to this moduledef handleEvent(self, event):@@ -100,8 +100,8 @@ class sfp_myspace(SpiderFootPlugin):# Check for email address as name, at the risk of missed results.try:- matches = re.findall('<a href=\"\/([a-zA-Z0-9_]+)\".*[\&; :\"\#\*\(\"\'\;\,\>\.\?\!]+' + email + '[\&; :\"\#\*\)\"\'\;\,\<\.\?\!]+', profile, re.IGNORECASE)- except BaseException as e:+ matches = re.findall(r'<a href=\"\/([a-zA-Z0-9_]+)\".*[\&; :\"\#\*\(\"\'\;\,\>\.\?\!]+' + email + r'[\&; :\"\#\*\)\"\'\;\,\<\.\?\!]+', profile, re.IGNORECASE)+ except BaseException:self.sf.debug("Malformed e-mail address, skipping.")return Nonediff --git a/modules/sfp_names.py b/modules/sfp_names.pyindex 99fc852f..3d97cba0 100644--- a/modules/sfp_names.py+++ b/modules/sfp_names.py@@ -12,7 +12,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_names(SpiderFootPlugin):"""Human Name Extractor:Footprint,Passive:Content Analysis:errorprone:Attempt to identify human names in fetched content."""@@ -20,9 +20,9 @@ class sfp_names(SpiderFootPlugin):meta = {'name': "Human Name Extractor",'summary': "Attempt to identify human names in fetched content.",- 'flags': [ "errorprone" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': ["errorprone"],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Content Analysis"]}# Default options@@ -106,7 +106,7 @@ class sfp_names(SpiderFootPlugin):# For RAW_RIR_DATA, there are only specific modules we# expect to see RELEVANT names within.if eventName == "RAW_RIR_DATA":- if srcModuleName not in [ "sfp_arin", "sfp_builtwith", "sfp_clearbit",+ if srcModuleName not in ["sfp_arin", "sfp_builtwith", "sfp_clearbit","sfp_fullcontact", "sfp_github", "sfp_hunter","sfp_opencorporates", "sfp_slideshare","sfp_twitter", "sfp_venmo", "sfp_instagram"]:@@ -134,7 +134,7 @@ class sfp_names(SpiderFootPlugin):# If both words are not in the dictionary, add 75 points.if first not in self.d and second not in self.d:- self.sf.debug("Both first and second names are not in the dictionary, so high chance of name: (" + first +":" + second +").")+ self.sf.debug(f"Both first and second names are not in the dictionary, so high chance of name: ({first}:{second}).")p += 75notindict = Trueelse:diff --git a/modules/sfp_networksdb.py b/modules/sfp_networksdb.pyindex b0cec144..88f8ce81 100644--- a/modules/sfp_networksdb.py+++ b/modules/sfp_networksdb.py@@ -14,7 +14,7 @@import jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_networksdb(SpiderFootPlugin):"""NetworksDB:Footprint,Investigate,Passive:Passive DNS:apikey:Search NetworksDB.io API for IP address and domain information."""@@ -22,9 +22,9 @@ class sfp_networksdb(SpiderFootPlugin):meta = {'name': "NetworksDB",'summary': "Search NetworksDB.io API for IP address and domain information.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "https://networksdb.io/",'model': "FREE_AUTH_LIMITED",@@ -233,7 +233,7 @@ class sfp_networksdb(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from NetworksDB.", False)+ self.sf.error(f"Error processing JSON response from NetworksDB: {e}", False)return Noneif data.get('warning'):diff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.pyindex 5f783f02..80c160f1 100644--- a/modules/sfp_neutrinoapi.py+++ b/modules/sfp_neutrinoapi.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_neutrinoapi(SpiderFootPlugin):"""NeutrinoAPI:Footprint,Investigate,Passive:Reputation Systems:apikey:Search NeutrinoAPI for IP address info and check IP reputation."""@@ -20,9 +20,9 @@ class sfp_neutrinoapi(SpiderFootPlugin):meta = {'name': "NeutrinoAPI",'summary': "Search NeutrinoAPI for IP address info and check IP reputation.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.neutrinoapi.com/",'model': "FREE_AUTH_LIMITED",@@ -82,36 +82,48 @@ class sfp_neutrinoapi(SpiderFootPlugin):# Query the phone-validate REST API# https://www.neutrinoapi.com/api/phone-validate/def queryPhoneValidate(self, qry):- res = self.sf.fetchUrl('https://neutrinoapi.com/phone-validate',- postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ 'https://neutrinoapi.com/phone-validate',+ postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},+ timeout=self.opts['timeout'],+ useragent=self.opts['_useragent']+ )return self.parseApiResponse(res)# Query the ip-info REST API# https://www.neutrinoapi.com/api/ip-info/def queryIpInfo(self, qry):- res = self.sf.fetchUrl("https://neutrinoapi.com/ip-info",- postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ "https://neutrinoapi.com/ip-info",+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},+ timeout=self.opts['timeout'],+ useragent=self.opts['_useragent']+ )return self.parseApiResponse(res)# Query the ip-blocklist REST API# https://www.neutrinoapi.com/api/ip-blocklist/def queryIpBlocklist(self, qry):- res = self.sf.fetchUrl("https://neutrinoapi.com/ip-blocklist",- postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ "https://neutrinoapi.com/ip-blocklist",+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},+ timeout=self.opts['timeout'],+ useragent=self.opts['_useragent']+ )return self.parseApiResponse(res)# Query the host-reputation REST API# https://www.neutrinoapi.com/api/host-reputation/def queryHostReputation(self, qry):- res = self.sf.fetchUrl("https://neutrinoapi.com/host-reputation",- postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},- timeout=self.opts['timeout'], useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(+ "https://neutrinoapi.com/host-reputation",+ postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},+ timeout=self.opts['timeout'],+ useragent=self.opts['_useragent']+ )return self.parseApiResponse(res)@@ -128,7 +140,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Noneif res['code'] == "400":diff --git a/modules/sfp_norton.py b/modules/sfp_norton.pyindex 0ce0aff1..a00c9eda 100644--- a/modules/sfp_norton.py+++ b/modules/sfp_norton.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_norton(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_norton(SpiderFootPlugin):meta = {'name': "Norton ConnectSafe",'summary': "Check if a host would be blocked by Norton ConnectSafe DNS",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ]+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"]}# Default options@@ -56,13 +56,13 @@ class sfp_norton(SpiderFootPlugin):def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "199.85.126.20", "199.85.127.20" ]+ res.nameservers = ["199.85.126.20", "199.85.127.20"]try:addrs = res.query(qaddr)self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -88,7 +88,7 @@ class sfp_norton(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:+ except BaseException:return Noneif resolved:diff --git a/modules/sfp_numverify.py b/modules/sfp_numverify.pyindex 63f8d744..84057a2c 100644--- a/modules/sfp_numverify.py+++ b/modules/sfp_numverify.py@@ -13,7 +13,7 @@import jsonimport urllib.request, urllib.parse, urllib.errorimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_numverify(SpiderFootPlugin):"""numverify:Footprint,Investigate,Passive:Real World:apikey:Lookup phone number location and carrier information from numverify.com."""@@ -21,9 +21,9 @@ class sfp_numverify(SpiderFootPlugin):meta = {'name': "numverify",'summary': "Lookup phone number location and carrier information from numverify.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Real World" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Real World"],'dataSource': {'website': "http://numverify.com/",'model': "FREE_AUTH_LIMITED",@@ -85,7 +85,7 @@ class sfp_numverify(SpiderFootPlugin):params = {'number': number.encode('raw_unicode_escape').decode("ascii", errors='replace'),'country_code': '',- 'format': '0', # set to "1" for prettified debug output+ 'format': '0', # set to "1" for prettified debug output'access_key': self.opts['api_key']}diff --git a/modules/sfp_onioncity.py b/modules/sfp_onioncity.pyindex ee7688fd..87be8e50 100644--- a/modules/sfp_onioncity.py+++ b/modules/sfp_onioncity.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport re@@ -21,9 +21,9 @@ class sfp_onioncity(SpiderFootPlugin):meta = {'name': "Onion.link",'summary': "Search Tor 'Onion City' search engine for mentions of the target domain.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Search Engines" ]+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Search Engines"]}# Default options@@ -117,7 +117,7 @@ class sfp_onioncity(SpiderFootPlugin):timeout=self.opts["_fetchtimeout"],useragent=self.opts["_useragent"],)- if response['code'] in [ "200", "201", "202" ]:+ if response['code'] in ["200", "201", "202"]:evt = SpiderFootEvent("RAW_RIR_DATA", response["content"], self.__name__, event)@@ -146,8 +146,8 @@ class sfp_onioncity(SpiderFootPlugin):continue# Sometimes onion city search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:self.sf.debug("Ignoring " + link + " as no mention of " + eventData)continue@@ -157,7 +157,7 @@ class sfp_onioncity(SpiderFootPlugin):try:startIndex = res['content'].index(eventData) - 120endIndex = startIndex + len(eventData) + 240- except BaseException as e:+ except BaseException:self.sf.debug("String not found in content.")continuediff --git a/modules/sfp_onionsearchengine.py b/modules/sfp_onionsearchengine.pyindex 64066efb..7197841a 100644--- a/modules/sfp_onionsearchengine.py+++ b/modules/sfp_onionsearchengine.py@@ -14,7 +14,7 @@import reimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_onionsearchengine(SpiderFootPlugin):"""Onionsearchengine.com:Footprint,Investigate:Search Engines::Search Tor onionsearchengine.com for mentions of the target domain."""@@ -22,9 +22,9 @@ class sfp_onionsearchengine(SpiderFootPlugin):meta = {'name': "Onionsearchengine.com",'summary': "Search Tor onionsearchengine.com for mentions of the target domain.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://as.onionsearchengine.com",'model': "FREE_NOAUTH_UNLIMITED",@@ -44,7 +44,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):'timeout': 10,'max_pages': 20,'fetchlinks': True,- 'blacklist': [ '.*://relate.*' ],+ 'blacklist': ['.*://relate.*'],'fullnames': True}@@ -78,7 +78,6 @@ class sfp_onionsearchengine(SpiderFootPlugin):def handleEvent(self, event):eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif not self.opts['fullnames'] and eventName == 'HUMAN_NAME':@@ -128,7 +127,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):self.__name__, event)self.notifyListeners(evt)- links = re.findall("url\.php\?u=(.[^\"\']+)[\"\']",+ links = re.findall(r"url\.php\?u=(.[^\"\']+)[\"\']",data['content'], re.IGNORECASE | re.DOTALL)for link in links:@@ -177,7 +176,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):try:startIndex = res['content'].index(eventData) - 120endIndex = startIndex + len(eventData) + 240- except BaseException as e:+ except BaseException:self.sf.debug('String "' + eventData + '" not found in content.')continuediff --git a/modules/sfp_onyphe.py b/modules/sfp_onyphe.pyindex 11af0371..3ca51d4b 100644--- a/modules/sfp_onyphe.py+++ b/modules/sfp_onyphe.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport jsonfrom datetime import datetimeimport timediff --git a/modules/sfp_open_passive_dns_database.py b/modules/sfp_open_passive_dns_database.pyindex 3a9dc756..cb95621b 100644--- a/modules/sfp_open_passive_dns_database.py+++ b/modules/sfp_open_passive_dns_database.py@@ -14,7 +14,7 @@import reimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_open_passive_dns_database(SpiderFootPlugin):"""Open Passive DNS Database:Footprint,Investigate,Passive:Passive DNS::Obtain passive DNS information from pdns.daloo.de Open passive DNS database."""@@ -22,9 +22,9 @@ class sfp_open_passive_dns_database(SpiderFootPlugin):meta = {'name': "Open Passive DNS Database",'summary': "Obtain passive DNS information from pdns.daloo.de Open passive DNS database.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "http://pdns.daloo.de/",'model': "FREE_NOAUTH_UNLIMITED",@@ -73,7 +73,7 @@ class sfp_open_passive_dns_database(SpiderFootPlugin):# Query the Open Passive DNS Database for a domaindef query(self, qry):params = {- "alike": 1, # alike is required to find subdomains+ "alike": 1, # alike is required to find subdomains"q": qry.encode('raw_unicode_escape').decode("ascii", errors='replace')}diff --git a/modules/sfp_openbugbounty.py b/modules/sfp_openbugbounty.pyindex 4a965d00..dc53dbdb 100644--- a/modules/sfp_openbugbounty.py+++ b/modules/sfp_openbugbounty.py@@ -11,7 +11,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_openbugbounty(SpiderFootPlugin):"""Open Bug Bounty:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed."""@@ -19,9 +19,9 @@ class sfp_openbugbounty(SpiderFootPlugin):meta = {'name': "Open Bug Bounty",'summary': "Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://www.openbugbounty.org/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.pyindex 700bfe09..cc6a988d 100644--- a/modules/sfp_opencorporates.py+++ b/modules/sfp_opencorporates.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_opencorporates(SpiderFootPlugin):"""OpenCorporates:Passive,Footprint,Investigate:Search Engines::Look up company information from OpenCorporates."""@@ -20,9 +20,9 @@ class sfp_opencorporates(SpiderFootPlugin):meta = {'name': "OpenCorporates",'summary': "Look up company information from OpenCorporates.",- 'flags': [ "" ],- 'useCases': [ "Passive", "Footprint", "Investigate" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Passive", "Footprint", "Investigate"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://opencorporates.com",'model': "FREE_NOAUTH_LIMITED",@@ -62,11 +62,11 @@ class sfp_opencorporates(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "COMPANY_NAME" ]+ return ["COMPANY_NAME"]# What events this module producesdef producedEvents(self):- return [ "COMPANY_NAME", "PHYSICAL_ADDRESS", "RAW_RIR_DATA" ]+ return ["COMPANY_NAME", "PHYSICAL_ADDRESS", "RAW_RIR_DATA"]# Search for company name# https://api.opencorporates.com/documentation/API-Referencediff --git a/modules/sfp_opendns.py b/modules/sfp_opendns.pyindex 2e952ced..8f297a40 100644--- a/modules/sfp_opendns.py+++ b/modules/sfp_opendns.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_opendns(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_opendns(SpiderFootPlugin):meta = {'name': "OpenDNS",'summary': "Check if a host would be blocked by OpenDNS DNS",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.opendns.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -71,13 +71,13 @@ class sfp_opendns(SpiderFootPlugin):def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "208.67.222.222", "208.67.220.220" ]+ res.nameservers = ["208.67.222.222", "208.67.220.220"]try:addrs = res.query(qaddr)self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -103,7 +103,7 @@ class sfp_opendns(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:+ except BaseException:return Noneif resolved:diff --git a/modules/sfp_openphish.py b/modules/sfp_openphish.pyindex 3de71e3c..e3d36a60 100644--- a/modules/sfp_openphish.py+++ b/modules/sfp_openphish.py@@ -13,14 +13,14 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'OpenPhish': {'id': '_openphish','checks': ['domain'],'url': 'https://www.openphish.com/feed.txt',- 'regex': '\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*'+ 'regex': r'\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*'}}@@ -31,9 +31,9 @@ class sfp_openphish(SpiderFootPlugin):meta = {'name': "OpenPhish",'summary': "Check if a host/domain is malicious according to OpenPhish.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://openphish.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -121,8 +121,7 @@ class sfp_openphish(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):@@ -140,11 +139,10 @@ class sfp_openphish(SpiderFootPlugin):try:if IPAddress(ip) in IPNetwork(target):- self.sf.debug(ip + " found within netblock/subnet " +- target + " in " + check)+ self.sf.debug(f"{ip} found within netblock/subnet {target} in {check}")return urlexcept Exception as e:- self.sf.debug("Error encountered parsing: " + str(e))+ self.sf.debug(f"Error encountered parsing: {e}")continuereturn Nonediff --git a/modules/sfp_openstreetmap.py b/modules/sfp_openstreetmap.pyindex 05586a9e..25d7af3d 100644--- a/modules/sfp_openstreetmap.py+++ b/modules/sfp_openstreetmap.py@@ -15,7 +15,7 @@ import reimport jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_openstreetmap(SpiderFootPlugin):"""OpenStreetMap:Footprint,Investigate,Passive:Real World::Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API."""@@ -23,9 +23,9 @@ class sfp_openstreetmap(SpiderFootPlugin):meta = {'name': "OpenStreetMap",'summary': "Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Real World" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Real World"],'dataSource': {'website': "https://www.openstreetmap.org/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.pyindex a860334b..9bd0f53c 100644--- a/modules/sfp_pageinfo.py+++ b/modules/sfp_pageinfo.py@@ -14,7 +14,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEvent# Indentify pages that use Javascript libs, handle passwords, have forms,# permit file uploads and more to come.@@ -34,9 +34,9 @@ class sfp_pageinfo(SpiderFootPlugin):meta = {'name': "Page Information",'summary': "Obtain information about web pages (do they take passwords, do they contain forms, etc.)",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.pyindex deb600c9..541f16d5 100644--- a/modules/sfp_pastebin.py+++ b/modules/sfp_pastebin.py@@ -13,7 +13,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_pastebin(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_pastebin(SpiderFootPlugin):meta = {'name': "PasteBin",'summary': "PasteBin scraping (via Google) to identify related content.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://pastebin.com/",'model': "FREE_AUTH_LIMITED",@@ -86,8 +86,6 @@ class sfp_pastebin(SpiderFootPlugin):return ["LEAKSITE_CONTENT", "LEAKSITE_URL"]def handleEvent(self, event):- eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif self.errorState:@@ -104,7 +102,6 @@ class sfp_pastebin(SpiderFootPlugin):self.results[eventData] = Truefor dom in list(self.domains.keys()):- links = list()target = self.domains[dom]res = self.sf.googleIterate(searchString="+site:{target_site} \"{search_keyword}\"".format(@@ -148,13 +145,13 @@ class sfp_pastebin(SpiderFootPlugin):continue# Sometimes pastes search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:continuetry:startIndex = res['content'].index(eventData)- except BaseException as e:+ except BaseException:self.sf.debug("String not found in pastes content.")continuediff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.pyindex 1e3c1a2a..c51e8d5a 100644--- a/modules/sfp_pgp.py+++ b/modules/sfp_pgp.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_pgp(SpiderFootPlugin):"""PGP Key Servers:Footprint,Investigate,Passive:Public Registries::Look up e-mail addresses in PGP public key servers."""@@ -21,9 +21,9 @@ class sfp_pgp(SpiderFootPlugin):meta = {'name': "PGP Key Servers",'summary': "Look up e-mail addresses in PGP public key servers.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Public Registries" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Public Registries"]}results = Nonediff --git a/modules/sfp_phishstats.py b/modules/sfp_phishstats.pyindex cde22328..3ea27841 100644--- a/modules/sfp_phishstats.py+++ b/modules/sfp_phishstats.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventfrom netaddr import IPNetworkimport urllib.request, urllib.parse, urllib.errorimport json@@ -22,9 +22,9 @@ class sfp_phishstats(SpiderFootPlugin):meta = {'name': "PhishStats",'summary': "Determine if an IP Address is malicious",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://phishstats.info/",'model': "FREE_NOAUTH_UNLIMITED",@@ -67,13 +67,21 @@ class sfp_phishstats(SpiderFootPlugin):# What events is this module interested in for input# For a list of all events, check sfdb.py.def watchedEvents(self):- return ["IP_ADDRESS", "NETBLOCK_OWNER", "NETBLOCK_MEMBER",- "AFFILIATE_IPADDR"]+ return [+ "IP_ADDRESS",+ "NETBLOCK_OWNER",+ "NETBLOCK_MEMBER",+ "AFFILIATE_IPADDR"+ ]# What events this module producesdef producedEvents(self):- return ["IP_ADDRESS", "MALICIOUS_IPADDR", "RAW_RIR_DATA",- "MALICIOUS_AFFILIATE_IPADDR"]+ return [+ "IP_ADDRESS",+ "MALICIOUS_IPADDR",+ "RAW_RIR_DATA",+ "MALICIOUS_AFFILIATE_IPADDR"+ ]# Check whether the IP Address is malicious using Phishstats API# https://phishstats.info/@@ -84,7 +92,7 @@ class sfp_phishstats(SpiderFootPlugin):}headers = {- 'Accept' : "application/json",+ 'Accept': "application/json",}res = self.sf.fetchUrl(@@ -126,22 +134,22 @@ class sfp_phishstats(SpiderFootPlugin):if eventName == 'NETBLOCK_OWNER':if not self.opts['netblocklookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:- self.sf.debug("Network size bigger than permitted: " +- str(IPNetwork(eventData).prefixlen) + " > " +- str(self.opts['maxnetblock']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxnetblock']))+ return Noneif eventName == 'NETBLOCK_MEMBER':if not self.opts['subnetlookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:- self.sf.debug("Network size bigger than permitted: " +- str(IPNetwork(eventData).prefixlen) + " > " +- str(self.opts['maxsubnet']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxsubnet']))+ return Noneqrylist = list()if eventName.startswith("NETBLOCK_"):diff --git a/modules/sfp_phishtank.py b/modules/sfp_phishtank.pyindex c0db17d8..2b4639ae 100644--- a/modules/sfp_phishtank.py+++ b/modules/sfp_phishtank.py@@ -13,14 +13,14 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'PhishTank': {'id': '_phishtank','checks': ['domain'],'url': 'http://data.phishtank.com/data/online-valid.csv',- 'regex': '\d+,\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*,http://www.phishtank.com/.*'+ 'regex': r'\d+,\w+://(.*\.)?[^a-zA-Z0-9]?{0}.*,http://www.phishtank.com/.*'}}@@ -31,9 +31,9 @@ class sfp_phishtank(SpiderFootPlugin):meta = {'name': "PhishTank",'summary': "Check if a host/domain is malicious according to PhishTank.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://phishtank.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -117,8 +117,7 @@ class sfp_phishtank(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_phone.py b/modules/sfp_phone.pyindex d6d11d67..e0c1d4cf 100644--- a/modules/sfp_phone.py+++ b/modules/sfp_phone.py@@ -15,7 +15,7 @@import phonenumbersfrom phonenumbers import carrier#from phonenumbers import geocoder-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_phone(SpiderFootPlugin):"""Phone Number Extractor:Passive,Footprint,Investigate:Content Analysis::Identify phone numbers in scraped webpages."""@@ -23,9 +23,9 @@ class sfp_phone(SpiderFootPlugin):meta = {'name': "Phone Number Extractor",'summary': "Identify phone numbers in scraped webpages.",- 'flags': [ "" ],- 'useCases': [ "Passive", "Footprint", "Investigate" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Passive", "Footprint", "Investigate"],+ 'categories': ["Content Analysis"]}# Default options@@ -70,8 +70,7 @@ class sfp_phone(SpiderFootPlugin):content = eventData.replace('.', '-')for match in phonenumbers.PhoneNumberMatcher(content, region=None):- n = phonenumbers.format_number(match.number,- phonenumbers.PhoneNumberFormat.E164)+ n = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)evt = SpiderFootEvent("PHONE_NUMBER", n, self.__name__, event)if event.moduleDataSource:evt.moduleDataSource = event.moduleDataSourcediff --git a/modules/sfp_psbdmp.py b/modules/sfp_psbdmp.pyindex 4869cce7..5f951125 100644--- a/modules/sfp_psbdmp.py+++ b/modules/sfp_psbdmp.py@@ -12,7 +12,7 @@import reimport json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_psbdmp(SpiderFootPlugin):"""Psbdmp:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Check psbdmp.cc (PasteBin Dump) for potentially hacked e-mails and domains."""@@ -20,9 +20,9 @@ class sfp_psbdmp(SpiderFootPlugin):meta = {'name': "Psbdmp",'summary': "Check psbdmp.cc (PasteBin Dump) for potentially hacked e-mails and domains.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://psbdmp.cc/",'model': "FREE_NOAUTH_UNLIMITED",@@ -81,8 +81,7 @@ class sfp_psbdmp(SpiderFootPlugin):else:url = "https://psbdmp.cc/api/search/domain/" + qry- res = self.sf.fetchUrl(url, timeout=15,- useragent="SpiderFoot")+ res = self.sf.fetchUrl(url, timeout=15, useragent="SpiderFoot")if res['code'] == "403" or res['content'] is None:self.sf.info("Unable to fetch data from psbdmp.cc right now.")@@ -137,13 +136,13 @@ class sfp_psbdmp(SpiderFootPlugin):continue# Sometimes pastes search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:continuetry:startIndex = res['content'].index(eventData)- except BaseException as e:+ except BaseException:self.sf.debug("String not found in pastes content.")continuediff --git a/modules/sfp_pulsedive.py b/modules/sfp_pulsedive.pyindex 60630c33..d7a5041f 100644--- a/modules/sfp_pulsedive.py+++ b/modules/sfp_pulsedive.py@@ -15,7 +15,7 @@ from datetime import datetimeimport timefrom netaddr import IPNetworkimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_pulsedive(SpiderFootPlugin):"""Pulsedive:Investigate,Passive:Reputation Systems:apikey:Obtain information from Pulsedive's API."""@@ -23,9 +23,9 @@ class sfp_pulsedive(SpiderFootPlugin):meta = {'name': "Pulsedive",'summary': "Obtain information from Pulsedive's API.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://pulsedive.com/",'model': "FREE_AUTH_LIMITED",@@ -119,7 +119,7 @@ class sfp_pulsedive(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from Pulsedive.", False)+ self.sf.error(f"Error processing JSON response from Pulsedive: {e}", False)return Nonereturn info@@ -226,7 +226,7 @@ class sfp_pulsedive(SpiderFootPlugin):if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:self.sf.debug("Record found but too old, skipping.")continue- except BaseException as e:+ except BaseException:self.sf.debug("Couldn't parse date from Pulsedive so assuming it's OK.")e = SpiderFootEvent(evtType, descr, self.__name__, event)self.notifyListeners(e)diff --git a/modules/sfp_quad9.py b/modules/sfp_quad9.pyindex a24ba473..099b5467 100644--- a/modules/sfp_quad9.py+++ b/modules/sfp_quad9.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_quad9(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_quad9(SpiderFootPlugin):meta = {'name': "Quad9",'summary': "Check if a host would be blocked by Quad9",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://quad9.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -72,13 +72,13 @@ class sfp_quad9(SpiderFootPlugin):def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "9.9.9.9" ]+ res.nameservers = ["9.9.9.9"]try:addrs = res.query(qaddr)self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -104,7 +104,7 @@ class sfp_quad9(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:+ except BaseException:return Noneif resolved:diff --git a/modules/sfp_riskiq.py b/modules/sfp_riskiq.pyindex 684affd3..3ae5e1c0 100644--- a/modules/sfp_riskiq.py+++ b/modules/sfp_riskiq.py@@ -12,7 +12,7 @@import jsonimport base64-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_riskiq(SpiderFootPlugin):"""RiskIQ:Investigate,Passive:Reputation Systems:apikey:Obtain information from RiskIQ's (formerly PassiveTotal) Passive DNS and Passive SSL databases."""@@ -20,9 +20,9 @@ class sfp_riskiq(SpiderFootPlugin):meta = {'name': "RiskIQ",'summary': "Obtain information from RiskIQ's (formerly PassiveTotal) Passive DNS and Passive SSL databases.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://community.riskiq.com/",'model': "FREE_AUTH_LIMITED",@@ -127,7 +127,7 @@ class sfp_riskiq(SpiderFootPlugin):useragent="SpiderFoot", headers=headers,postData=post)- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("RiskIQ access seems to have been rejected or you have exceeded usage limits.", False)self.errorState = Truereturn None@@ -142,7 +142,7 @@ class sfp_riskiq(SpiderFootPlugin):self.sf.info("No RiskIQ info found for " + qry)return Noneexcept BaseException as e:- self.sf.error("Invalid JSON returned by RiskIQ.", False)+ self.sf.error(f"Invalid JSON returned by RiskIQ: {e}", False)return Nonereturn ret['results']@@ -176,7 +176,7 @@ class sfp_riskiq(SpiderFootPlugin):else:self.results[eventData] = True- if eventName in [ 'DOMAIN_NAME' ]:+ if eventName in ['DOMAIN_NAME']:ret = self.query(eventData, "PSSL")if not ret:self.sf.info("No RiskIQ passive SSL data found for " + eventData)@@ -220,7 +220,7 @@ class sfp_riskiq(SpiderFootPlugin):return None- if eventName in [ 'IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME' ]:+ if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']:ret = self.query(eventData, "PDNS")if not ret:self.sf.info("No RiskIQ passive DNS data found for " + eventData)@@ -237,7 +237,7 @@ class sfp_riskiq(SpiderFootPlugin):# We found a co-hostcohosts.append(r['focusPoint'])- if eventName in [ "INTERNET_NAME", "DOMAIN_NAME" ]:+ if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]:# Record could be an A/CNAME of this entity, or something pointing to itfor r in ret:if r['focusPoint'].endswith("."):diff --git a/modules/sfp_robtex.py b/modules/sfp_robtex.pyindex e6f3318c..a0c5fea8 100644--- a/modules/sfp_robtex.py+++ b/modules/sfp_robtex.py@@ -13,7 +13,7 @@import jsonimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_robtex(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_robtex(SpiderFootPlugin):meta = {'name': "Robtex",'summary': "Search Robtex.com for hosts sharing the same IP.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "https://www.robtex.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -85,7 +85,7 @@ class sfp_robtex(SpiderFootPlugin):# Don't notify me about events from myselfdef watchOpts(self):- return [ 'noself' ]+ return ['noself']# Handle events sent to this moduledef handleEvent(self, event):@@ -137,16 +137,13 @@ class sfp_robtex(SpiderFootPlugin):qrylist.append(eventData)self.results[eventData] = True- myres = list()-for ip in qrylist:if self.checkForStop():return Noneretry = 0while retry < 2:- res = self.sf.fetchUrl("https://freeapi.robtex.com/ipquery/" + ip,- timeout=self.opts['_fetchtimeout'])+ res = self.sf.fetchUrl("https://freeapi.robtex.com/ipquery/" + ip, timeout=self.opts['_fetchtimeout'])if res['code'] == "200":breakif res['code'] == "404":@@ -164,7 +161,7 @@ class sfp_robtex(SpiderFootPlugin):try:data = json.loads(res['content'])except BaseException as e:- self.sf.error("Error parsing JSON from robtex API.", False)+ self.sf.error(f"Error parsing JSON from Robtex API: {e}", False)return Nonepas = data.get('pas')diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.pyindex ca441d21..5fb4e8ec 100644--- a/modules/sfp_s3bucket.py+++ b/modules/sfp_s3bucket.py@@ -13,7 +13,7 @@import threadingimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_s3bucket(SpiderFootPlugin):"""Amazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents."""@@ -21,9 +21,9 @@ class sfp_s3bucket(SpiderFootPlugin):meta = {'name': "Amazon S3 Bucket Finder",'summary': "Search for potential Amazon S3 buckets associated with the target and attempt to list their contents.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Crawling and Scanning" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Crawling and Scanning"],'dataSource': {'website': "https://aws.amazon.com/s3/",'model': "FREE_NOAUTH_UNLIMITED",@@ -70,7 +70,7 @@ class sfp_s3bucket(SpiderFootPlugin):def checkSite(self, url):res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)- if res['code'] not in [ "301", "302", "200" ] and \+ if res['code'] not in ["301", "302", "200"] and \(res['content'] is None or "NoSuchBucket" in res['content']):self.sf.debug("Not a valid bucket: " + url)else:@@ -82,7 +82,6 @@ class sfp_s3bucket(SpiderFootPlugin):self.s3results[url] = 0def threadSites(self, siteList):- ret = list()self.s3results = dict()running = Truei = 0@@ -156,7 +155,7 @@ class sfp_s3bucket(SpiderFootPlugin):self.notifyListeners(evt)return None- targets = [ eventData.replace('.', '') ]+ targets = [eventData.replace('.', '')]kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])if kw:targets.append(kw)diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.pyindex 1be9a0f7..d212ac73 100644--- a/modules/sfp_scylla.py+++ b/modules/sfp_scylla.py@@ -13,7 +13,7 @@ import jsonimport timeimport base64import urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_scylla(SpiderFootPlugin):"""Scylla:Footprint,Investigate,Passive:Leaks, Dumps and Breaches::Gather breach data from Scylla API."""@@ -21,9 +21,9 @@ class sfp_scylla(SpiderFootPlugin):meta = {'name': "Scylla",'summary': "Gather breach data from Scylla API.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://scylla.sh/",'model': "FREE_NOAUTH_UNLIMITED",@@ -47,8 +47,8 @@ class sfp_scylla(SpiderFootPlugin):# Option descriptionsoptdescs = {- 'pause': "Number of seconds to pause between fetches.",- 'per_page': "Maximum number of results per page.",+ 'pause': "Number of seconds to pause between fetches.",+ 'per_page': "Maximum number of results per page.",'max_pages': "Maximum number of pages of results to fetch."}@@ -65,11 +65,11 @@ class sfp_scylla(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ 'DOMAIN_NAME' ]+ return ['DOMAIN_NAME']# What events this module producesdef producedEvents(self):- return [ 'EMAILADDR_COMPROMISED', 'PASSWORD_COMPROMISED', 'HASH_COMPROMISED', 'RAW_RIR_DATA' ]+ return ['EMAILADDR_COMPROMISED', 'PASSWORD_COMPROMISED', 'HASH_COMPROMISED', 'RAW_RIR_DATA']# Query Scylla APIdef query(self, qry, per_page=20, start=0):diff --git a/modules/sfp_securitytrails.py b/modules/sfp_securitytrails.pyindex 8d69529b..6836b01f 100644--- a/modules/sfp_securitytrails.py+++ b/modules/sfp_securitytrails.py@@ -12,7 +12,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_securitytrails(SpiderFootPlugin):"""SecurityTrails:Investigate,Passive:Search Engines:apikey:Obtain Passive DNS and other information from SecurityTrails"""@@ -20,9 +20,9 @@ class sfp_securitytrails(SpiderFootPlugin):meta = {'name': "SecurityTrails",'summary': "Obtain Passive DNS and other information from SecurityTrails",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://securitytrails.com/",'model': "FREE_AUTH_LIMITED",@@ -111,7 +111,7 @@ class sfp_securitytrails(SpiderFootPlugin):useragent="SpiderFoot", headers=headers,postData=request)- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("SecurityTrails API key seems to have been rejected or you have exceeded usage limits for the month.", False)self.errorState = Truereturn None@@ -166,7 +166,7 @@ class sfp_securitytrails(SpiderFootPlugin):else:self.results[eventData] = True- if eventName in [ "IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]:+ if eventName in ["IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]:ip = eventDatarec = self.query(ip, "ipv4")myres = list()@@ -200,7 +200,7 @@ class sfp_securitytrails(SpiderFootPlugin):self.notifyListeners(e)self.cohostcount += 1- if eventName in [ "EMAILADDR"]:+ if eventName in ["EMAILADDR"]:email = eventDatarec = self.query(email, "whois_email")myres = list()diff --git a/modules/sfp_shodan.py b/modules/sfp_shodan.pyindex 9a257b36..2085a0c1 100644--- a/modules/sfp_shodan.py+++ b/modules/sfp_shodan.py@@ -14,7 +14,7 @@ import jsonimport urllib.request, urllib.parse, urllib.errorimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_shodan(SpiderFootPlugin):@@ -23,9 +23,9 @@ class sfp_shodan(SpiderFootPlugin):meta = {'name': "SHODAN",'summary': "Obtain information from SHODAN about identified IP addresses.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.shodan.io/",'model': "FREE_AUTH_LIMITED",@@ -97,7 +97,7 @@ class sfp_shodan(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from SHODAN.", False)+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)return Nonereturn info@@ -114,7 +114,7 @@ class sfp_shodan(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from SHODAN.", False)+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)return Nonereturn info@@ -135,7 +135,7 @@ class sfp_shodan(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from SHODAN.", False)+ self.sf.error(f"Error processing JSON response from SHODAN: {e}", False)return Nonereturn infodiff --git a/modules/sfp_similar.py b/modules/sfp_similar.pyindex e90c4666..f057b396 100644--- a/modules/sfp_similar.py+++ b/modules/sfp_similar.py@@ -11,50 +11,50 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventnearchars = {- 'a': [ '4', 's' ],- 'b': [ 'v', 'n' ],- 'c': [ 'x', 'v' ],- 'd': [ 's', 'f' ],- 'e': [ 'w', 'r' ],- 'f': [ 'd', 'g' ],- 'g': [ 'f', 'h' ],- 'h': [ 'g', 'j', 'n' ],- 'i': [ 'o', 'u', '1' ],- 'j': [ 'k', 'h', 'i' ],- 'k': [ 'l', 'j' ],- 'l': [ 'i', '1', 'k' ],- 'm': [ 'n' ],- 'n': [ 'm' ],- 'o': [ 'p', 'i', '0' ],- 'p': [ 'o', 'q' ],- 'r': [ 't', 'e' ],- 's': [ 'a', 'd', '5' ],- 't': [ '7', 'y', 'z', 'r' ],- 'u': [ 'v', 'i', 'y', 'z' ],- 'v': [ 'u', 'c', 'b' ],- 'w': [ 'v', 'vv', 'q', 'e' ],- 'x': [ 'z', 'y', 'c' ],- 'y': [ 'z', 'x' ],- 'z': [ 'y', 'x' ],- '0': [ 'o' ],- '1': [ 'l' ],- '2': [ '5' ],- '3': [ 'e' ],- '4': [ 'a' ],- '5': [ 's' ],- '6': [ 'b' ],- '7': [ 't' ],- '8': [ 'b' ],+ 'a': ['4', 's'],+ 'b': ['v', 'n'],+ 'c': ['x', 'v'],+ 'd': ['s', 'f'],+ 'e': ['w', 'r'],+ 'f': ['d', 'g'],+ 'g': ['f', 'h'],+ 'h': ['g', 'j', 'n'],+ 'i': ['o', 'u', '1'],+ 'j': ['k', 'h', 'i'],+ 'k': ['l', 'j'],+ 'l': ['i', '1', 'k'],+ 'm': ['n'],+ 'n': ['m'],+ 'o': ['p', 'i', '0'],+ 'p': ['o', 'q'],+ 'r': ['t', 'e'],+ 's': ['a', 'd', '5'],+ 't': ['7', 'y', 'z', 'r'],+ 'u': ['v', 'i', 'y', 'z'],+ 'v': ['u', 'c', 'b'],+ 'w': ['v', 'vv', 'q', 'e'],+ 'x': ['z', 'y', 'c'],+ 'y': ['z', 'x'],+ 'z': ['y', 'x'],+ '0': ['o'],+ '1': ['l'],+ '2': ['5'],+ '3': ['e'],+ '4': ['a'],+ '5': ['s'],+ '6': ['b'],+ '7': ['t'],+ '8': ['b'],'9': []}pairs = {- 'oo': [ '00' ],- 'll': [ 'l1l', 'l1l', '111', '11' ],- '11': [ 'll', 'lll', 'l1l', '1l1' ]+ 'oo': ['00'],+ 'll': ['l1l', 'l1l', '111', '11'],+ '11': ['ll', 'lll', 'l1l', '1l1']}# These are all string builders, {0} = the domain keyword, {1} = the page number@@ -64,9 +64,9 @@ class sfp_similar(SpiderFootPlugin):meta = {'name': "Similar Domain Finder",'summary': "Search various sources to identify similar looking domain names, for instance squatted domains.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "DNS" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["DNS"]}# Default options@@ -100,8 +100,6 @@ class sfp_similar(SpiderFootPlugin):# Search for similar sounding domainsdef handleEvent(self, event):- eventName = event.eventType- srcModuleName = event.moduleeventData = event.datadomlist = list()@@ -160,7 +158,7 @@ class sfp_similar(SpiderFootPlugin):self.sf.debug("Resolved " + d + tld)evt = SpiderFootEvent("SIMILARDOMAIN", d + tld, self.__name__, event)self.notifyListeners(evt)- except BaseException as e:+ except BaseException:continuereturn Nonediff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.pyindex c4caf2de..7ddefa89 100644--- a/modules/sfp_skymem.py+++ b/modules/sfp_skymem.py@@ -13,7 +13,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_skymem(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_skymem(SpiderFootPlugin):meta = {'name': "Skymem",'summary': "Look up e-mail addresses on Skymem.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "http://www.skymem.info/",'model': "FREE_NOAUTH_UNLIMITED",@@ -138,7 +138,7 @@ class sfp_skymem(SpiderFootPlugin):# Check if we're on the last page of resultsmax_page = 0- pages = re.findall(r'/domain/' + domain_id + '\?p=(\d+)', res['content'])+ pages = re.findall(r'/domain/' + domain_id + r'\?p=(\d+)', res['content'])for p in pages:if int(p) >= max_page:max_page = int(p)diff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.pyindex f818ad21..67f4ea03 100644--- a/modules/sfp_slideshare.py+++ b/modules/sfp_slideshare.py@@ -11,7 +11,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_slideshare(SpiderFootPlugin):"""SlideShare:Footprint,Investigate,Passive:Social Media::Gather name and location from SlideShare profiles."""@@ -19,9 +19,9 @@ class sfp_slideshare(SpiderFootPlugin):meta = {'name': "SlideShare",'summary': "Gather name and location from SlideShare profiles.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://www.slideshare.net",'model': "FREE_NOAUTH_UNLIMITED",@@ -57,15 +57,15 @@ class sfp_slideshare(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "SOCIAL_MEDIA" ]+ return ["SOCIAL_MEDIA"]# What events this module producesdef producedEvents(self):- return [ "RAW_RIR_DATA", "GEOINFO" ]+ return ["RAW_RIR_DATA", "GEOINFO"]# Extract meta property contents from HTMLdef extractMeta(self, meta_property, html):- return re.findall(r'<meta property="' + meta_property + '"\s+content="(.+)" />', html)+ return re.findall(r'<meta property="' + meta_property + r'"\s+content="(.+)" />', html)# Handle events sent to this moduledef handleEvent(self, event):diff --git a/modules/sfp_snov.py b/modules/sfp_snov.pyindex 0d1c5d63..67fb79fa 100644--- a/modules/sfp_snov.py+++ b/modules/sfp_snov.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport urllib.request, urllib.parse, urllib.errorimport json@@ -21,9 +21,9 @@ class sfp_snov(SpiderFootPlugin):meta = {'name': "Snov",'summary': "Gather available email IDs from identified domains",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://snov.io/",'model': "FREE_AUTH_LIMITED",@@ -127,7 +127,7 @@ class sfp_snov(SpiderFootPlugin):}headers = {- 'Accept' : "application/json",+ 'Accept': "application/json",}res = self.sf.fetchUrl(diff --git a/modules/sfp_social.py b/modules/sfp_social.pyindex 6f29f94a..1f538d93 100644--- a/modules/sfp_social.py+++ b/modules/sfp_social.py@@ -12,21 +12,21 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventregexps = dict({"LinkedIn (Individual)": list(['.*linkedin.com/in/([a-zA-Z0-9_]+$)']),"LinkedIn (Company)": list(['.*linkedin.com/company/([a-zA-Z0-9_]+$)']),- "Github": list(['.*github.com/([a-zA-Z0-9_]+)\/']),+ "Github": list([r'.*github.com/([a-zA-Z0-9_]+)\/']),"Google+": list(['.*plus.google.com/([0-9]+$)']),"Facebook": list(['.*facebook.com/([a-zA-Z0-9_]+$)']),- "MySpace": list(['https?://myspace.com/([a-zA-Z0-9_\.]+$)']),+ "MySpace": list([r'https?://myspace.com/([a-zA-Z0-9_\.]+$)']),"YouTube": list(['.*youtube.com/([a-zA-Z0-9_]+$)']),"Twitter": list(['.*twitter.com/([a-zA-Z0-9_]{1,15}$)','.*twitter.com/#!/([a-zA-Z0-9_]{1,15}$)']),"SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)']),- "Instagram": list(['.*instagram.com/([a-zA-Z0-9_\.]+)/?$'])+ "Instagram": list([r'.*instagram.com/([a-zA-Z0-9_\.]+)/?$'])})@@ -36,9 +36,9 @@ class sfp_social(SpiderFootPlugin):meta = {'name': "Social Network Identifier",'summary': "Identify presence on social media networks such as LinkedIn, Twitter and others.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Social Media" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Social Media"]}# Default options@@ -98,7 +98,7 @@ class sfp_social(SpiderFootPlugin):# Except for Google+, the link includes potential usernamesif regexpGrp != "Google+":un = bits.group(1)- evt = SpiderFootEvent("USERNAME", bits.group(1), self.__name__, event)+ evt = SpiderFootEvent("USERNAME", str(un), self.__name__, event)self.notifyListeners(evt)return Nonediff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.pyindex fe705eae..e1238c9a 100644--- a/modules/sfp_socialprofiles.py+++ b/modules/sfp_socialprofiles.py@@ -13,7 +13,7 @@import reimport urllib.request, urllib.error, urllib.parse-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventsites = {# Search string to use, domain name the profile will sit on within@@ -21,22 +21,22 @@ sites = {"Facebook": ["\"{name}\"+site:facebook.com",[- '[ \'"](https?://[a-z\.]*facebook.[a-z\.]+/[^/"\'<> ]+/?)[\'" ]',- '(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/"\'<> ]+/?)',+ r'[ \'"](https?://[a-z\.]*facebook.[a-z\.]+/[^/"\'<> ]+/?)[\'"]',+ r'(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/"\'<> ]+/?)',],],"Google+": ["\"{name}\"+site:plus.google.com",[- '[ \'"](https?://plus.google.[a-z\.]+/\d+[^"\'<>\/ ]+)[\'" ]',- '(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/"\'<> ]+)',+ r'[ \'"](https?://plus.google.[a-z\.]+/\d+[^"\'<>\/ ]+)[\'"]',+ r'(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/"\'<> ]+)',],],"LinkedIn": ["\"{name}\"+site:linkedin.com",[- '["\' ](https?://[a-z\.]*linkedin.[a-z\.]+/[^\?"\'<> ]+)[\'" ]',- '(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\?"\'<> ]+)',+ r'["\' ](https?://[a-z\.]*linkedin.[a-z\.]+/[^\?"\'<> ]+)[\'"]',+ r'(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\?"\'<> ]+)',],],}@@ -48,9 +48,9 @@ class sfp_socialprofiles(SpiderFootPlugin):meta = {'name': "Social Media Profile Finder",'summary': "Tries to discover the social media profiles for human names identified.",- 'flags': [ "slow", "apikey" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Social Media" ]+ 'flags': ["slow", "apikey"],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Social Media"]}# Default options@@ -219,7 +219,7 @@ class sfp_socialprofiles(SpiderFootPlugin):found = Falsefor kw in self.keywords:if re.search(- "[^a-zA-Z\-\_]" + kw + "[^a-zA-Z\-\_]",+ r"[^a-zA-Z\-\_]" + kw + r"[^a-zA-Z\-\_]",pres["content"],re.IGNORECASE,):diff --git a/modules/sfp_sorbs.py b/modules/sfp_sorbs.pyindex dc5aa493..0e25388b 100644--- a/modules/sfp_sorbs.py+++ b/modules/sfp_sorbs.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_sorbs(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_sorbs(SpiderFootPlugin):meta = {'name': "SORBS",'summary': "Query the SORBS database for open relays, open proxies, vulnerable servers, etc.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.sorbs.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -154,7 +154,6 @@ class sfp_sorbs(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataparentEvent = event- addrlist = list()self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")diff --git a/modules/sfp_spamcop.py b/modules/sfp_spamcop.pyindex a49d0817..dc95b454 100644--- a/modules/sfp_spamcop.py+++ b/modules/sfp_spamcop.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_spamcop(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_spamcop(SpiderFootPlugin):meta = {'name': "SpamCop",'summary': "Query various spamcop databases for open relays, open proxies, vulnerable servers, etc.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.spamcop.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -145,7 +145,6 @@ class sfp_spamcop(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataparentEvent = event- addrlist = list()self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")diff --git a/modules/sfp_spamhaus.py b/modules/sfp_spamhaus.pyindex 0a4b5fae..3a94befd 100644--- a/modules/sfp_spamhaus.py+++ b/modules/sfp_spamhaus.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_spamhaus(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_spamhaus(SpiderFootPlugin):meta = {'name': "Spamhaus",'summary': "Query the Spamhaus databases for open relays, open proxies, vulnerable servers, etc.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.spamhaus.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -157,7 +157,6 @@ class sfp_spamhaus(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataparentEvent = event- addrlist = list()self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex 493b5e41..3a48b756 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -13,7 +13,7 @@import timeimport json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_spider(SpiderFootPlugin):"""Web Spider:Footprint,Investigate:Crawling and Scanning:slow:Spidering of web-pages to extract content for searching."""@@ -21,9 +21,9 @@ class sfp_spider(SpiderFootPlugin):meta = {'name': "Web Spider",'summary': "Spidering of web-pages to extract content for searching.",- 'flags': [ "slow" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Crawling and Scanning" ]+ 'flags': ["slow"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Crawling and Scanning"]}# Default options@@ -250,7 +250,7 @@ class sfp_spider(SpiderFootPlugin):# Don't notify me about events from myselfdef watchOpts(self):- return [ 'noself' ]+ return ['noself']# What events this module produces# This is to support the end user in selecting modules based on eventsdiff --git a/modules/sfp_spur.py b/modules/sfp_spur.pyindex c6992c86..539fa26a 100644--- a/modules/sfp_spur.py+++ b/modules/sfp_spur.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventfrom netaddr import IPNetworkimport json@@ -21,9 +21,9 @@ class sfp_spur(SpiderFootPlugin):meta = {'name': "spur.us",'summary': "Obtain information about any malicious activities involving IP addresses found",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://spur.us/",'model': "COMMERCIAL ONLY",@@ -78,13 +78,23 @@ class sfp_spur(SpiderFootPlugin):# What events is this module interested in for input# For a list of all events, check sfdb.py.def watchedEvents(self):- return ["IP_ADDRESS", "NETBLOCK_OWNER", "NETBLOCK_MEMBER",- "AFFILIATE_IPADDR"]+ return [+ "IP_ADDRESS",+ "NETBLOCK_OWNER",+ "NETBLOCK_MEMBER",+ "AFFILIATE_IPADDR"+ ]# What events this module producesdef producedEvents(self):- return ["IP_ADDRESS", "MALICIOUS_IPADDR", "RAW_RIR_DATA",- "GEO_INFO", "COMPANY_NAME", "MALICIOUS_AFFILIATE_IPADDR"]+ return [+ "IP_ADDRESS",+ "MALICIOUS_IPADDR",+ "RAW_RIR_DATA",+ "GEO_INFO",+ "COMPANY_NAME",+ "MALICIOUS_AFFILIATE_IPADDR"+ ]# Check whether the IP Address is malicious using spur.us API# https://spur.us/app/docs@@ -143,29 +153,28 @@ class sfp_spur(SpiderFootPlugin):if eventData in self.results:self.sf.debug(f"Skipping {eventData}, already checked.")return None- else:- self.results[eventData] = True+ self.results[eventData] = Trueif eventName == 'NETBLOCK_OWNER':if not self.opts['netblocklookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:- self.sf.debug("Network size bigger than permitted: " +- str(IPNetwork(eventData).prefixlen) + " > " +- str(self.opts['maxnetblock']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxnetblock']))+ return Noneif eventName == 'NETBLOCK_MEMBER':if not self.opts['subnetlookup']:return None- else:- if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:- self.sf.debug("Network size bigger than permitted: " +- str(IPNetwork(eventData).prefixlen) + " > " +- str(self.opts['maxsubnet']))- return None++ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxsubnet']))+ return Noneqrylist = list()if eventName.startswith("NETBLOCK_"):@@ -250,7 +259,7 @@ class sfp_spur(SpiderFootPlugin):if vpnOperatorsExists:vpnOperatorNames = vpnOperators.get('operators')- maliciousIPDesc = "spur.us [ " + str(addr) + " ]\n"+ maliciousIPDesc = "spur.us [" + str(addr) + "]\n"maliciousIPDesc += "VPN Operators : "for operatorNameDict in vpnOperatorNames:diff --git a/modules/sfp_spyonweb.py b/modules/sfp_spyonweb.pyindex 54ce0413..c470abc6 100644--- a/modules/sfp_spyonweb.py+++ b/modules/sfp_spyonweb.py@@ -14,7 +14,7 @@import datetimeimport jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_spyonweb(SpiderFootPlugin):"""SpyOnWeb:Footprint,Investigate,Passive:Passive DNS:apikey:Search SpyOnWeb for hosts sharing the same IP address, Google Analytics code, or Google Adsense code."""@@ -22,9 +22,9 @@ class sfp_spyonweb(SpiderFootPlugin):meta = {'name': "SpyOnWeb",'summary': "Search SpyOnWeb for hosts sharing the same IP address, Google Analytics code, or Google Adsense code.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "http://spyonweb.com/",'model': "FREE_AUTH_LIMITED",@@ -105,7 +105,7 @@ class sfp_spyonweb(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonestatus = data.get('status')@@ -212,7 +212,7 @@ class sfp_spyonweb(SpiderFootPlugin):agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])# Find Google AdSense IDs and Google Analytics IDs for the specified domain- if eventName in [ 'INTERNET_NAME', 'DOMAIN_NAME' ]:+ if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:data = self.querySummary(eventData, limit=self.opts['limit'])if data is None:@@ -234,7 +234,7 @@ class sfp_spyonweb(SpiderFootPlugin):self.notifyListeners(evt)# Find affiliate domains for the specified Google AdSense ID or Google Analytics ID- if eventName in [ 'WEB_ANALYTICS_ID' ]:+ if eventName in ['WEB_ANALYTICS_ID']:try:network = eventData.split(": ")[0]analytics_id = eventData.split(": ")[1]@@ -270,14 +270,14 @@ class sfp_spyonweb(SpiderFootPlugin):self.notifyListeners(evt)# Find co-hosts on the same IP address- if eventName in [ 'IP_ADDRESS' ]:+ if eventName in ['IP_ADDRESS']:data = self.queryIP(eventData, limit=self.opts['limit'])if data is None:self.sf.info("No data found for " + eventData)return None- cohostcount = 0+ self.cohostcount = 0for co in list(data.keys()):last_seen = int(datetime.datetime.strptime(data[co], '%Y-%m-%d').strftime('%s')) * 1000diff --git a/modules/sfp_spyse.py b/modules/sfp_spyse.pyindex 3423accc..9bb5c790 100644--- a/modules/sfp_spyse.py+++ b/modules/sfp_spyse.py@@ -15,7 +15,7 @@import jsonimport timeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_spyse(SpiderFootPlugin):"""Spyse:Footprint,Investigate,Passive:Passive DNS:apikey:SpiderFoot plug-in to search Spyse API for IP address and domain information."""@@ -23,9 +23,9 @@ class sfp_spyse(SpiderFootPlugin):meta = {'name': "Spyse",'summary': "SpiderFoot plug-in to search Spyse API for IP address and domain information.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Passive DNS" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],'dataSource': {'website': "https://spyse.com",'model': "FREE_AUTH_LIMITED",@@ -102,8 +102,8 @@ class sfp_spyse(SpiderFootPlugin):'offset': currentOffset}headers = {- 'Accept' : "application/json",- 'Authorization' : "Bearer " + self.opts['api_key']+ 'Accept': "application/json",+ 'Authorization': "Bearer " + self.opts['api_key']}res = self.sf.fetchUrl(@@ -126,8 +126,8 @@ class sfp_spyse(SpiderFootPlugin):'offset': currentOffset}headers = {- 'Accept' : "application/json",- 'Authorization' : "Bearer " + self.opts['api_key']+ 'Accept': "application/json",+ 'Authorization': "Bearer " + self.opts['api_key']}res = self.sf.fetchUrl('https://api.spyse.com/v3/data/ip/port?' + urllib.parse.urlencode(params),@@ -149,8 +149,8 @@ class sfp_spyse(SpiderFootPlugin):'offset': currentOffset}headers = {- 'Accept' : "application/json",- 'Authorization' : "Bearer " + self.opts['api_key']+ 'Accept': "application/json",+ 'Authorization': "Bearer " + self.opts['api_key']}res = self.sf.fetchUrl('https://api.spyse.com/v3/data/ip/domain?' + urllib.parse.urlencode(params),@@ -174,8 +174,8 @@ class sfp_spyse(SpiderFootPlugin):}headers = {- 'Accept' : "application/json",- 'Authorization' : "Bearer " + self.opts['api_key']+ 'Accept': "application/json",+ 'Authorization': "Bearer " + self.opts['api_key']}res = self.sf.fetchUrl(@@ -224,7 +224,7 @@ class sfp_spyse(SpiderFootPlugin):try:data = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Noneif data.get('message'):diff --git a/modules/sfp_sslcert.py b/modules/sfp_sslcert.pyindex 81a0d754..bb156587 100644--- a/modules/sfp_sslcert.py+++ b/modules/sfp_sslcert.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------from urllib.parse import urlparse-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_sslcert(SpiderFootPlugin):@@ -20,9 +20,9 @@ class sfp_sslcert(SpiderFootPlugin):meta = {'name': "SSL Certificate Analyzer",'summary': "Gather information about SSL certificates used by the target's HTTPS sites.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Crawling and Scanning" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Crawling and Scanning"]}# Default options@@ -91,7 +91,7 @@ class sfp_sslcert(SpiderFootPlugin):if u.port:port = u.portfqdn = self.sf.urlFQDN(eventData.lower())- except BaseException as e:+ except BaseException:self.sf.debug("Couldn't parse URL: " + eventData)return Noneelse:diff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.pyindex 62149df8..66ae8d52 100644--- a/modules/sfp_strangeheaders.py+++ b/modules/sfp_strangeheaders.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEvent# Standard headers, taken from http://en.wikipedia.org/wiki/List_of_HTTP_header_fieldsheaders = ["access-control-allow-origin", "accept-ranges", "age", "allow", "cache-control",@@ -30,9 +30,9 @@ class sfp_strangeheaders(SpiderFootPlugin):meta = {'name': "Strange Header Identifier",'summary': "Obtain non-standard HTTP headers returned by web servers.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Content Analysis"]}# Default options@@ -80,7 +80,7 @@ class sfp_strangeheaders(SpiderFootPlugin):jdata = json.loads(eventData)if jdata == None:return None- except BaseException as e:+ except BaseException:self.sf.error("Received HTTP headers from another module in an unexpected format.", False)return Nonediff --git a/modules/sfp_subdomain_takeover.py b/modules/sfp_subdomain_takeover.pyindex ec7da1b3..1bb1b549 100644--- a/modules/sfp_subdomain_takeover.py+++ b/modules/sfp_subdomain_takeover.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_subdomain_takeover(SpiderFootPlugin):"""Subdomain Takeover:Footprint,Investigate:Crawling and Scanning::Check if affiliated subdomains are vulnerable to takeover."""@@ -21,9 +21,9 @@ class sfp_subdomain_takeover(SpiderFootPlugin):meta = {'name': "Subdomain Takeover",'summary': "Check if affiliated subdomains are vulnerable to takeover.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Crawling and Scanning" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Crawling and Scanning"]}# Default options@@ -63,7 +63,7 @@ class sfp_subdomain_takeover(SpiderFootPlugin):try:self.fingerprints = json.loads(content)except Exception as e:- self.sf.error("Unable to parse subdomain takeover fingerprints list.", False)+ self.sf.error(f"Unable to parse subdomain takeover fingerprints list: {e}", False)self.errorState = Truereturn Nonediff --git a/modules/sfp_talosintel.py b/modules/sfp_talosintel.pyindex a16ea71e..30f34905 100644--- a/modules/sfp_talosintel.py+++ b/modules/sfp_talosintel.py@@ -13,7 +13,7 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'talosintelligence.com List': {@@ -30,9 +30,9 @@ class sfp_talosintel(SpiderFootPlugin):meta = {'name': "Talos Intelligence",'summary': "Check if a netblock or IP is malicious according to talosintelligence.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://talosintelligence.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -126,8 +126,7 @@ class sfp_talosintel(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_template.py b/modules/sfp_template.pyindex 8493088b..26502215 100644--- a/modules/sfp_template.py+++ b/modules/sfp_template.py@@ -62,13 +62,13 @@ class sfp_template(SpiderFootPlugin):# - errorprone: Might generate high false positives# - invasive: Interrogates the target, might be intensive# - tool: Runs an external tool to collect data- 'flags': [ "slow", "apikey"],+ 'flags': ["slow", "apikey"],# Use cases: The use case(s) this module should be included in, options are Footprint, Investigate and Passive.# - Passive means the user's scan target is not contacted at all# - Footprint means that this module is useful when understanding the target's footprint on the Internet# - Investigate means that this module is useful when investigating the danger/risk of a target- 'useCases': [ "Passive" ],+ 'useCases': ["Passive"],# Categories: The categories this module belongs in, describing how it operates. Only the first category is# used for now.@@ -83,7 +83,7 @@ class sfp_template(SpiderFootPlugin):# - Search Engines: Searches public search engines with data about the whole Internet# - Secondary Networks: Queries information about participation on secondary networks, like Bitcoin# - Social Media: Searches social media data sources- 'categories': [ "Social Media" ],+ 'categories': ["Social Media"],# For tool modules, have some basic information about the tool.'toolDetails': {diff --git a/modules/sfp_threatcrowd.py b/modules/sfp_threatcrowd.pyindex 59fb8be0..e8f55eba 100644--- a/modules/sfp_threatcrowd.py+++ b/modules/sfp_threatcrowd.py@@ -12,7 +12,7 @@import jsonfrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_threatcrowd(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_threatcrowd(SpiderFootPlugin):meta = {'name': "ThreatCrowd",'summary': "Obtain information from ThreatCrowd about identified IP addresses, domains and e-mail addresses.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.threatcrowd.org",'model': "FREE_NOAUTH_UNLIMITED",@@ -110,7 +110,7 @@ class sfp_threatcrowd(SpiderFootPlugin):try:ret = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from ThreatCrowd.", False)+ self.sf.error(f"Error processing JSON response from ThreatCrowd: {e}", False)self.errorState = Truereturn Nonediff --git a/modules/sfp_threatminer.py b/modules/sfp_threatminer.pyindex dee93aba..768ecc4b 100644--- a/modules/sfp_threatminer.py+++ b/modules/sfp_threatminer.py@@ -14,7 +14,7 @@ import jsonfrom datetime import datetimeimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_threatminer(SpiderFootPlugin):"""ThreatMiner:Footprint,Investigate,Passive:Search Engines::Obtain information from ThreatMiner's database for passive DNS and threat intelligence."""@@ -22,9 +22,9 @@ class sfp_threatminer(SpiderFootPlugin):meta = {'name': "ThreatMiner",'summary': "Obtain information from ThreatMiner's database for passive DNS and threat intelligence.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.threatminer.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -119,7 +119,7 @@ class sfp_threatminer(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from ThreatMiner.", False)+ self.sf.error(f"Error processing JSON response from ThreatMiner: {e}", False)return Nonereturn infodiff --git a/modules/sfp_tldsearch.py b/modules/sfp_tldsearch.pyindex 8c31005c..a1fd1259 100644--- a/modules/sfp_tldsearch.py+++ b/modules/sfp_tldsearch.py@@ -15,7 +15,7 @@ import randomimport threadingimport timeimport dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_tldsearch(SpiderFootPlugin):"""TLD Searcher:Footprint:DNS:slow:Search all Internet TLDs for domains with the same name as the target (this can be very slow.)"""@@ -23,9 +23,9 @@ class sfp_tldsearch(SpiderFootPlugin):meta = {'name': "TLD Searcher",'summary': "Search all Internet TLDs for domains with the same name as the target (this can be very slow.)",- 'flags': [ "slow" ],- 'useCases': [ "Footprint" ],- 'categories': [ "DNS" ]+ 'flags': ["slow"],+ 'useCases': ["Footprint"],+ 'categories': ["DNS"]}# Default options@@ -87,7 +87,7 @@ class sfp_tldsearch(SpiderFootPlugin):else:with self.lock:self.tldResults[target] = True- except BaseException as e:+ except BaseException:with self.lock:self.tldResults[target] = False@@ -146,8 +146,6 @@ class sfp_tldsearch(SpiderFootPlugin):# Search for similar sounding domainsdef handleEvent(self, event):- eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif eventData in self.results:diff --git a/modules/sfp_tool_cmseek.py b/modules/sfp_tool_cmseek.pyindex 9ac03bc7..04c44850 100644--- a/modules/sfp_tool_cmseek.py+++ b/modules/sfp_tool_cmseek.py@@ -15,7 +15,7 @@ from subprocess import Popen, PIPEimport ioimport jsonimport os.path-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_tool_cmseek(SpiderFootPlugin):"""Tool - CMSeeK:Footprint,Investigate:Content Analysis:tool:Identify what Content Management System (CMS) might be used."""@@ -23,9 +23,9 @@ class sfp_tool_cmseek(SpiderFootPlugin):meta = {'name': "Tool - CMSeeK",'summary': "Identify what Content Management System (CMS) might be used.",- 'flags': [ "tool" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Content Analysis" ],+ 'flags': ["tool"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Content Analysis"],'toolDetails': {'name': "CMSeeK",'description': "CMSeek is a tool that is used to extract Content Management System(CMS) details of a website.",diff --git a/modules/sfp_tool_dnstwist.py b/modules/sfp_tool_dnstwist.pyindex 706c416f..4aa13a75 100644--- a/modules/sfp_tool_dnstwist.py+++ b/modules/sfp_tool_dnstwist.py@@ -14,7 +14,7 @@from subprocess import Popen, PIPEimport jsonimport os.path-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_tool_dnstwist(SpiderFootPlugin):"""Tool - DNSTwist:Footprint,Investigate:DNS:tool:Identify bit-squatting, typo and other similar domains to the target using a local DNSTwist installation."""@@ -22,9 +22,9 @@ class sfp_tool_dnstwist(SpiderFootPlugin):meta = {'name': "Tool - DNSTwist",'summary': "Identify bit-squatting, typo and other similar domains to the target using a local DNSTwist installation.",- 'flags': [ "tool" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "DNS" ],+ 'flags': ["tool"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["DNS"],'toolDetails': {'name': "DNSTwist",'description': "See what sort of trouble users can get in trying to type your domain name. "diff --git a/modules/sfp_tool_nmap.py b/modules/sfp_tool_nmap.pyindex c72d2514..9d562bbc 100644--- a/modules/sfp_tool_nmap.py+++ b/modules/sfp_tool_nmap.py@@ -13,7 +13,7 @@from subprocess import Popen, PIPEimport os.pathfrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_tool_nmap(SpiderFootPlugin):"""Tool - Nmap:Footprint,Investigate:Crawling and Scanning:tool,slow,invasive:Identify what Operating System might be used."""@@ -21,9 +21,9 @@ class sfp_tool_nmap(SpiderFootPlugin):meta = {'name': "Tool - Nmap",'summary': "Identify what Operating System might be used.",- 'flags': [ "tool", "slow", "invasive" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Crawling and Scanning" ],+ 'flags': ["tool", "slow", "invasive"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Crawling and Scanning"],'toolDetails': {'name': "Nmap",'description': "Nmap (\"Network Mapper\") is a free and open source utility for network discovery and security auditing.\n"@@ -165,8 +165,7 @@ class sfp_tool_nmap(SpiderFootPlugin):if "OS details:" in line:junk, opsys = line.split(": ")if opsys:- evt = SpiderFootEvent("OPERATING_SYSTEM", opsys,- self.__name__, event)+ evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, event)self.notifyListeners(evt)except BaseException as e:self.sf.error("Couldn't parse the output of Nmap: " + str(e), False)@@ -183,16 +182,14 @@ class sfp_tool_nmap(SpiderFootPlugin):junk, opsys = line.split(": ")if opsys and currentIp:- ipevent = SpiderFootEvent("IP_ADDRESS", currentIp,- self.__name__, event)+ ipevent = SpiderFootEvent("IP_ADDRESS", currentIp, self.__name__, event)self.notifyListeners(ipevent)- evt = SpiderFootEvent("OPERATING_SYSTEM", opsys,- self.__name__, ipevent)+ evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, ipevent)self.notifyListeners(evt)currentIp = Noneexcept BaseException as e:- self.sf.error("Couldn't parse the output of Nmap: " + str(e), False)+ self.sf.error(f"Couldn't parse the output of Nmap: {e}", False)return None# End of sfp_tool_nmap classdiff --git a/modules/sfp_tool_whatweb.py b/modules/sfp_tool_whatweb.pyindex 88dafa7c..2c17e5a9 100644--- a/modules/sfp_tool_whatweb.py+++ b/modules/sfp_tool_whatweb.py@@ -14,7 +14,7 @@from subprocess import Popen, PIPEimport jsonimport os.path-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_tool_whatweb(SpiderFootPlugin):"""Tool - WhatWeb:Footprint,Investigate:Content Analysis:tool:Identify what software is in use on the specified website."""@@ -22,9 +22,9 @@ class sfp_tool_whatweb(SpiderFootPlugin):meta = {'name': "Tool - WhatWeb",'summary': "Identify what software is in use on the specified website.",- 'flags': [ "tool" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Content Analysis" ],+ 'flags': ["tool"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Content Analysis"],'toolDetails': {'name': "WhatWeb",'description': "WhatWeb identifies websites. Its goal is to answer the question, \"What is that Website?\". "diff --git a/modules/sfp_torch.py b/modules/sfp_torch.pyindex f23133cc..fe067575 100644--- a/modules/sfp_torch.py+++ b/modules/sfp_torch.py@@ -11,7 +11,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport re@@ -21,9 +21,9 @@ class sfp_torch(SpiderFootPlugin):meta = {'name': "TORCH",'summary': "Search Tor 'TORCH' search engine for mentions of the target domain.",- 'flags': [ "errorprone" ],- 'useCases': [ "Footprint", "Investigate" ],- 'categories': [ "Search Engines" ]+ 'flags': ["errorprone"],+ 'useCases': ["Footprint", "Investigate"],+ 'categories': ["Search Engines"]}# Default options@@ -62,7 +62,6 @@ class sfp_torch(SpiderFootPlugin):def handleEvent(self, event):eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataif not self.opts['fullnames'] and eventName == 'HUMAN_NAME':@@ -71,24 +70,26 @@ class sfp_torch(SpiderFootPlugin):if eventData in self.results:self.sf.debug("Already did a search for " + eventData + ", skipping.")return None- else:- self.results[eventData] = True- formpage = self.sf.fetchUrl("http://xmh57jrzrnw6insl.onion",- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout'])+ self.results[eventData] = True++ formpage = self.sf.fetchUrl(+ "http://xmh57jrzrnw6insl.onion",+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']+ )if not formpage['content']:self.sf.info("Couldn't connect to TORCH, check that you have TOR enabled.")return None- else:- # Need the form ID to submit later for the search- m = re.findall("\<form method=\"get\" action=\"/(\S+)/search.cgi\"\>",- formpage['content'], re.IGNORECASE | re.DOTALL)- if not m:- return None- formid = m[0]+ # Need the form ID to submit later for the search+ m = re.findall(r"\<form method=\"get\" action=\"/(\S+)/search.cgi\"\>",+ formpage['content'], re.IGNORECASE | re.DOTALL)+ if not m:+ return None++ formid = m[0]pagecontent = ""pagecount = 0@@ -119,7 +120,7 @@ class sfp_torch(SpiderFootPlugin):self.__name__, event)self.notifyListeners(evt)- links = re.findall("\<DT\>\d+.\s+<a href=\"(.*?)\"\s+TARGET=\"_blank\"\>",+ links = re.findall(r"\<DT\>\d+.\s+<a href=\"(.*?)\"\s+TARGET=\"_blank\"\>",data['content'], re.IGNORECASE | re.DOTALL)for link in links:@@ -149,7 +150,7 @@ class sfp_torch(SpiderFootPlugin):try:startIndex = res['content'].index(eventData) - 120endIndex = startIndex + len(eventData) + 240- except BaseException as e:+ except BaseException:self.sf.debug("String not found in content.")continuediff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.pyindex 709544d2..78d3a3ff 100644--- a/modules/sfp_torexits.py+++ b/modules/sfp_torexits.py@@ -13,14 +13,14 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'TOR Exist List': {'id': '_torexits','checks': ['ip', 'netblock'],'url': 'https://check.torproject.org/exit-addresses',- 'regex': '^ExitAddress\s+{0}\s+'+ 'regex': r'^ExitAddress\s+{0}\s+'}}@@ -31,9 +31,9 @@ class sfp_torexits(SpiderFootPlugin):meta = {'name': "TOR Exit Nodes",'summary': "Check if an IP or netblock appears on the torproject.org exit node list.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Secondary Networks" ]+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Secondary Networks"]}# Default options@@ -111,8 +111,7 @@ class sfp_torexits(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_totalhash.py b/modules/sfp_totalhash.pyindex 09eb279d..f29ae8d5 100644--- a/modules/sfp_totalhash.py+++ b/modules/sfp_totalhash.py@@ -12,7 +12,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'TotalHash.com Database': {@@ -31,9 +31,9 @@ class sfp_totalhash(SpiderFootPlugin):meta = {'name': "TotalHash.com",'summary': "Check if a host/domain or IP is malicious according to TotalHash.com.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://totalhash.cymru.com/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_twilio.py b/modules/sfp_twilio.pyindex 2243d360..fd530dcc 100644--- a/modules/sfp_twilio.py+++ b/modules/sfp_twilio.py@@ -10,7 +10,7 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventimport base64import json@@ -20,9 +20,9 @@ class sfp_twilio(SpiderFootPlugin):meta = {'name': "Twilio",'summary': "Obtain information from Twilio about phone numbers. Ensure you have the Caller Name add-on installed in Twilio.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.twilio.com/",'model': "FREE_AUTH_LIMITED",diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.pyindex a5de6135..9d5b6a23 100644--- a/modules/sfp_twitter.py+++ b/modules/sfp_twitter.py@@ -11,7 +11,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_twitter(SpiderFootPlugin):"""Twitter:Footprint,Investigate,Passive:Social Media::Gather name and location from Twitter profiles."""@@ -19,9 +19,9 @@ class sfp_twitter(SpiderFootPlugin):meta = {'name': "Twitter",'summary': "Gather name and location from Twitter profiles.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://twitter.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -52,11 +52,11 @@ class sfp_twitter(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ "SOCIAL_MEDIA" ]+ return ["SOCIAL_MEDIA"]# What events this module producesdef producedEvents(self):- return [ "RAW_RIR_DATA", "GEOINFO" ]+ return ["RAW_RIR_DATA", "GEOINFO"]# Handle events sent to this moduledef handleEvent(self, event):diff --git a/modules/sfp_uceprotect.py b/modules/sfp_uceprotect.pyindex baab3c19..cd17fcc0 100644--- a/modules/sfp_uceprotect.py+++ b/modules/sfp_uceprotect.py@@ -13,7 +13,7 @@# -------------------------------------------------------------------------------from netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_uceprotect(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_uceprotect(SpiderFootPlugin):meta = {'name': "UCEPROTECT",'summary': "Query the UCEPROTECT databases for open relays, open proxies, vulnerable servers, etc.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.uceprotect.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -144,7 +144,6 @@ class sfp_uceprotect(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.dataparentEvent = event- addrlist = list()self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")diff --git a/modules/sfp_urlscan.py b/modules/sfp_urlscan.pyindex 5be82df2..7f28472a 100644--- a/modules/sfp_urlscan.py+++ b/modules/sfp_urlscan.py@@ -12,7 +12,7 @@import jsonimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_urlscan(SpiderFootPlugin):"""URLScan.io:Footprint,Investigate,Passive:Search Engines::Search URLScan.io cache for domain information."""@@ -20,9 +20,9 @@ class sfp_urlscan(SpiderFootPlugin):meta = {'name': "URLScan.io",'summary': "Search URLScan.io cache for domain information.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://urlscan.io/",'model': "FREE_NOAUTH_UNLIMITED",@@ -91,7 +91,7 @@ class sfp_urlscan(SpiderFootPlugin):try:result = json.loads(res['content'])except Exception as e:- self.sf.debug("Error processing JSON response.")+ self.sf.debug(f"Error processing JSON response: {e}")return Nonereturn resultdiff --git a/modules/sfp_venmo.py b/modules/sfp_venmo.pyindex 6269bc95..8cae6cc1 100644--- a/modules/sfp_venmo.py+++ b/modules/sfp_venmo.py@@ -11,7 +11,7 @@import jsonimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_venmo(SpiderFootPlugin):"""Venmo:Footprint,Investigate,Passive:Social Media::Gather user information from Venmo API."""@@ -19,9 +19,9 @@ class sfp_venmo(SpiderFootPlugin):meta = {'name': "Venmo",'summary': "Gather user information from Venmo API.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Social Media" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Social Media"],'dataSource': {'website': "https://venmo.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -51,11 +51,11 @@ class sfp_venmo(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return [ 'USERNAME' ]+ return ['USERNAME']# What events this module producesdef producedEvents(self):- return [ 'RAW_RIR_DATA' ]+ return ['RAW_RIR_DATA']# Query Venmo APIdef query(self, qry):diff --git a/modules/sfp_viewdns.py b/modules/sfp_viewdns.pyindex 25ce5718..30fad380 100644--- a/modules/sfp_viewdns.py+++ b/modules/sfp_viewdns.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_viewdns(SpiderFootPlugin):"""ViewDNS.info:Investigate,Passive:Search Engines:apikey:Reverse Whois lookups using ViewDNS.info."""@@ -19,9 +19,9 @@ class sfp_viewdns(SpiderFootPlugin):meta = {'name': "ViewDNS.info",'summary': "Reverse Whois lookups using ViewDNS.info.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://viewdns.info/",'model': "FREE_AUTH_LIMITED",@@ -108,7 +108,7 @@ class sfp_viewdns(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot")- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("ViewDNS.info API key seems to have been rejected or you have exceeded usage limits.", False)self.errorState = Truereturn None@@ -196,7 +196,7 @@ class sfp_viewdns(SpiderFootPlugin):myres.append(h.lower())else:continue- if h.lower() in [ "demo1.com", "demo2.com", "demo3.com", "demo4.com", "demo5.com" ]:+ if h.lower() in ["demo1.com", "demo2.com", "demo3.com", "demo4.com", "demo5.com"]:continueif eventName == "EMAILADDR":e = SpiderFootEvent("AFFILIATE_INTERNET_NAME", h, self.__name__, event)diff --git a/modules/sfp_virustotal.py b/modules/sfp_virustotal.pyindex 3fca9192..d5374486 100644--- a/modules/sfp_virustotal.py+++ b/modules/sfp_virustotal.py@@ -13,7 +13,7 @@import jsonimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_virustotal(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_virustotal(SpiderFootPlugin):meta = {'name': "VirusTotal",'summary': "Obtain information from VirusTotal about identified IP addresses.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://www.virustotal.com/",'model': "FREE_AUTH_LIMITED",@@ -123,7 +123,7 @@ class sfp_virustotal(SpiderFootPlugin):try:ret = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from VirusTotal.", False)+ self.sf.error(f"Error processing JSON response from VirusTotal: {e}", False)self.errorState = Truereturn None@@ -226,7 +226,7 @@ class sfp_virustotal(SpiderFootPlugin):# Treat siblings as affiliates if they are of the original target, otherwise# they are additional hosts within the target.if 'domain_siblings' in info:- if eventName in [ "IP_ADDRESS", "INTERNET_NAME"]:+ if eventName in ["IP_ADDRESS", "INTERNET_NAME"]:for s in info['domain_siblings']:if self.getTarget().matches(s):if s not in self.results:diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.pyindex 7e06350f..df9368a6 100644--- a/modules/sfp_voipbl.py+++ b/modules/sfp_voipbl.py@@ -13,14 +13,14 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'VOIPBL Publicly Accessible PBX List': {'id': '_voipbl','checks': ['ip', 'netblock'],'url': 'http://www.voipbl.org/update',- 'regex': '{0}\/'+ 'regex': r'{0}\/'}}@@ -31,9 +31,9 @@ class sfp_voipbl(SpiderFootPlugin):meta = {'name': "VoIPBL OpenPBX IPs",'summary': "Check if an IP or netblock is an open PBX according to VoIPBL OpenPBX IPs.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://www.voipbl.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -123,8 +123,7 @@ class sfp_voipbl(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.pyindex 5df2a70d..4ffd78d4 100644--- a/modules/sfp_vxvault.py+++ b/modules/sfp_vxvault.py@@ -13,14 +13,14 @@from netaddr import IPAddress, IPNetworkimport re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'VXVault Malicious URL List': {'id': '_vxvault','checks': ['ip', 'domain'],'url': 'http://vxvault.net/URL_List.php',- 'regex': '.*\/{0}/.*'+ 'regex': r'.*\/{0}/.*'}}@@ -31,9 +31,9 @@ class sfp_vxvault(SpiderFootPlugin):meta = {'name': "VXVault.net",'summary': "Check if a domain or IP is malicious according to VXVault.net.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://vxvault.net/",'model': "FREE_NOAUTH_UNLIMITED",@@ -124,8 +124,7 @@ class sfp_vxvault(SpiderFootPlugin):# build a list of IP.# Cycle through each IP and check if it's in the netblock.if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")+ rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")pat = re.compile(rx, re.IGNORECASE)self.sf.debug("New regex for " + check + ": " + rx)for line in data['content'].split('\n'):diff --git a/modules/sfp_watchguard.py b/modules/sfp_watchguard.pyindex 1597bccf..14d0f6f3 100644--- a/modules/sfp_watchguard.py+++ b/modules/sfp_watchguard.py@@ -12,12 +12,12 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventmalchecks = {'Watchguard Reputation Authority Lookup': {'id': '_watchguard',- 'checks': [ 'ip' ],+ 'checks': ['ip'],'url': 'http://reputationauthority.org/lookup?ip={0}','badregex': ['.*>[6-9][0-9]/100 </td>.*', '.*>100/100 </td>.*'],'goodregex': []@@ -31,9 +31,9 @@ class sfp_watchguard(SpiderFootPlugin):meta = {'name': "Watchguard",'summary': "Check if an IP is malicious according to Watchguard's reputationauthority.org.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "http://reputationauthority.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -82,7 +82,7 @@ class sfp_watchguard(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR" ]+ return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR"]# Check the regexps to see whether the content indicates maliciousnessdef contentMalicious(self, content, goodregex, badregex):diff --git a/modules/sfp_webanalytics.py b/modules/sfp_webanalytics.pyindex 819da742..45163cda 100644--- a/modules/sfp_webanalytics.py+++ b/modules/sfp_webanalytics.py@@ -13,7 +13,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_webanalytics(SpiderFootPlugin):"""Web Analytics Extractor:Footprint,Investigate,Passive:Content Analysis::Identify web analytics IDs in scraped webpages and DNS TXT records."""@@ -21,9 +21,9 @@ class sfp_webanalytics(SpiderFootPlugin):meta = {'name': "Web Analytics Extractor",'summary': "Identify web analytics IDs in scraped webpages and DNS TXT records.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}opts = {}diff --git a/modules/sfp_webframework.py b/modules/sfp_webframework.pyindex 09361539..d9bfcef6 100644--- a/modules/sfp_webframework.py+++ b/modules/sfp_webframework.py@@ -12,18 +12,18 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventregexps = dict({"jQuery": list(['jquery']), # unlikely false positive- "YUI": list(['\/yui\/', 'yui\-', 'yui\.']),- "Prototype": list(['\/prototype\/', 'prototype\-', 'prototype\.js']),- "ZURB Foundation": list(['\/foundation\/', 'foundation\-', 'foundation\.js']),- "Bootstrap": list(['\/bootstrap\/', 'bootstrap\-', 'bootstrap\.js']),- "ExtJS": list(['[\'\"\=]ext\.js', 'extjs', '\/ext\/*\.js']),- "Mootools": list(['\/mootools\/', 'mootools\-', 'mootools\.js']),- "Dojo": list(['\/dojo\/', '[\'\"\=]dojo\-', '[\'\"\=]dojo\.js']),- "Wordpress": list(['\/wp-includes\/', '\/wp-content\/'])+ "YUI": list([r'\/yui\/', r'yui\-', r'yui\.']),+ "Prototype": list([r'\/prototype\/', r'prototype\-', r'prototype\.js']),+ "ZURB Foundation": list([r'\/foundation\/', r'foundation\-', r'foundation\.js']),+ "Bootstrap": list([r'\/bootstrap\/', r'bootstrap\-', r'bootstrap\.js']),+ "ExtJS": list([r'[\'\"\=]ext\.js', 'extjs', r'\/ext\/*\.js']),+ "Mootools": list([r'\/mootools\/', r'mootools\-', r'mootools\.js']),+ "Dojo": list([r'\/dojo\/', r'[\'\"\=]dojo\-', r'[\'\"\=]dojo\.js']),+ "Wordpress": list([r'\/wp-includes\/', r'\/wp-content\/'])})class sfp_webframework(SpiderFootPlugin):@@ -32,9 +32,9 @@ class sfp_webframework(SpiderFootPlugin):meta = {'name': "Web Framework Identifier",'summary': "Identify the usage of popular web frameworks like jQuery, YUI and others.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.pyindex 436c2a16..246af5da 100644--- a/modules/sfp_webserver.py+++ b/modules/sfp_webserver.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_webserver(SpiderFootPlugin):"""Web Server Identifier:Footprint,Investigate,Passive:Content Analysis::Obtain web server banners to identify versions of web servers being used."""@@ -20,9 +20,9 @@ class sfp_webserver(SpiderFootPlugin):meta = {'name': "Web Server Identifier",'summary': "Obtain web server banners to identify versions of web servers being used.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default options@@ -71,7 +71,7 @@ class sfp_webserver(SpiderFootPlugin):jdata = json.loads(eventData)if jdata == None:return None- except BaseException as e:+ except BaseException:self.sf.error("Received HTTP headers from another module in an unexpected format.", False)return None@@ -115,7 +115,7 @@ class sfp_webserver(SpiderFootPlugin):return Nonetech = None- if 'set-cookie'in jdata and 'PHPSESS' in jdata['set-cookie']:+ if 'set-cookie' in jdata and 'PHPSESS' in jdata['set-cookie']:tech = "PHP"if 'set-cookie' in jdata and 'JSESSIONID' in jdata['set-cookie']:diff --git a/modules/sfp_whatcms.py b/modules/sfp_whatcms.pyindex 52b292ad..e8b29d5e 100644--- a/modules/sfp_whatcms.py+++ b/modules/sfp_whatcms.py@@ -13,7 +13,7 @@import jsonimport urllib.request, urllib.parse, urllib.errorimport time-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_whatcms(SpiderFootPlugin):"""WhatCMS:Footprint,Investigate,Passive:Content Analysis:apikey,slow:Check web technology using WhatCMS.org API."""@@ -21,9 +21,9 @@ class sfp_whatcms(SpiderFootPlugin):meta = {'name': "WhatCMS",'summary': "Check web technology using WhatCMS.org API.",- 'flags': [ "apikey", "slow" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Content Analysis" ]+ 'flags': ["apikey", "slow"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Content Analysis"]}# Default optionsdiff --git a/modules/sfp_whois.py b/modules/sfp_whois.pyindex c6c4ea89..2d7dcbd2 100644--- a/modules/sfp_whois.py+++ b/modules/sfp_whois.py@@ -14,7 +14,7 @@import whoisimport ipwhoisfrom netaddr import IPAddress-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_whois(SpiderFootPlugin):"""Whois:Footprint,Investigate,Passive:Public Registries::Perform a WHOIS look-up on domain names and owned netblocks."""@@ -22,9 +22,9 @@ class sfp_whois(SpiderFootPlugin):meta = {'name': "Whois",'summary': "Perform a WHOIS look-up on domain names and owned netblocks.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Public Registries" ]+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Public Registries"]}# Default options@@ -47,7 +47,7 @@ class sfp_whois(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):return ["DOMAIN_NAME", "DOMAIN_NAME_PARENT", "NETBLOCK_OWNER",- "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN_NAME", "SIMILARDOMAIN" ]+ "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN_NAME", "SIMILARDOMAIN"]# What events this module produces# This is to support the end user in selecting modules based on eventsdiff --git a/modules/sfp_whoisology.py b/modules/sfp_whoisology.pyindex 9d321681..b6e59d1e 100644--- a/modules/sfp_whoisology.py+++ b/modules/sfp_whoisology.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_whoisology(SpiderFootPlugin):"""Whoisology:Investigate,Passive:Search Engines:apikey:Reverse Whois lookups using Whoisology.com."""@@ -19,9 +19,9 @@ class sfp_whoisology(SpiderFootPlugin):meta = {'name': "Whoisology",'summary': "Reverse Whois lookups using Whoisology.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://whoisology.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -81,7 +81,7 @@ class sfp_whoisology(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot")- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("Whoisology API key seems to have been rejected or you have exceeded usage limits.", False)self.errorState = Truereturn Nonediff --git a/modules/sfp_whoxy.py b/modules/sfp_whoxy.pyindex f2903917..7ded0fee 100644--- a/modules/sfp_whoxy.py+++ b/modules/sfp_whoxy.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import json-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_whoxy(SpiderFootPlugin):"""Whoxy:Investigate,Passive:Search Engines:apikey:Reverse Whois lookups using Whoxy.com."""@@ -19,9 +19,9 @@ class sfp_whoxy(SpiderFootPlugin):meta = {'name': "Whoxy",'summary': "Reverse Whois lookups using Whoxy.com.",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Search Engines" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Search Engines"],'dataSource': {'website': "https://www.whoxy.com/",'model': "COMMERCIAL_ONLY",@@ -89,7 +89,7 @@ class sfp_whoxy(SpiderFootPlugin):res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot")- if res['code'] in [ "400", "429", "500", "403" ]:+ if res['code'] in ["400", "429", "500", "403"]:self.sf.error("Whoxy API key seems to have been rejected or you have exceeded usage limits.", False)self.errorState = Truereturn Nonediff --git a/modules/sfp_wigle.py b/modules/sfp_wigle.pyindex 8ec5c519..c70aa797 100644--- a/modules/sfp_wigle.py+++ b/modules/sfp_wigle.py@@ -12,17 +12,17 @@import jsonimport datetimeimport urllib.request, urllib.parse, urllib.error-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_wigle(SpiderFootPlugin):- """Wigle.net:Footprint,Investigate,Passive:Secondary Networks:apikey:Query wigle.net to identify nearby WiFi access points."""+ """WiGLE:Footprint,Investigate,Passive:Secondary Networks:apikey:Query WiGLE to identify nearby WiFi access points."""meta = {- 'name': "Wigle.net",- 'summary': "Query wigle.net to identify nearby WiFi access points.",- 'flags': [ "apikey" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Secondary Networks" ],+ 'name': "WiGLE",+ 'summary': "Query WiGLE to identify nearby WiFi access points.",+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Secondary Networks"],'dataSource': {'website': "https://wigle.net/",'model': "FREE_AUTH_UNLIMITED",@@ -107,7 +107,7 @@ class sfp_wigle(SpiderFootPlugin):return Nonereturn info['results'][0]['boundingbox']except Exception as e:- self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)+ self.sf.error(f"Error processing JSON response from Wigle.net: {e}", False)return Nonedef getnetworks(self, coords):@@ -144,7 +144,7 @@ class sfp_wigle(SpiderFootPlugin):ret.append(r['ssid'] + " (Net ID: " + r['netid'] + ")")return retexcept Exception as e:- self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)+ self.sf.error(f"Error processing JSON response from WiGLE: {e}", False)return Nonediff --git a/modules/sfp_wikileaks.py b/modules/sfp_wikileaks.pyindex 9e821899..83b00fb4 100644--- a/modules/sfp_wikileaks.py+++ b/modules/sfp_wikileaks.py@@ -11,7 +11,7 @@# -------------------------------------------------------------------------------import datetime-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_wikileaks(SpiderFootPlugin):@@ -20,9 +20,9 @@ class sfp_wikileaks(SpiderFootPlugin):meta = {'name': "Wikileaks",'summary': "Search Wikileaks for mentions of domain names and e-mail addresses.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://wikileaks.org/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_wikipediaedits.py b/modules/sfp_wikipediaedits.pyindex e287c217..d4ad9d73 100644--- a/modules/sfp_wikipediaedits.py+++ b/modules/sfp_wikipediaedits.py@@ -14,7 +14,7 @@ import datetimeimport reimport urllib.request, urllib.parse, urllib.errorfrom html.parser import HTMLParser-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_wikipediaedits(SpiderFootPlugin):"""Wikipedia Edits:Footprint,Investigate,Passive:Secondary Networks::Identify edits to Wikipedia articles made from a given IP address or username."""@@ -22,9 +22,9 @@ class sfp_wikipediaedits(SpiderFootPlugin):meta = {'name': "Wikipedia Edits",'summary': "Identify edits to Wikipedia articles made from a given IP address or username.",- 'flags': [ "" ],- 'useCases': [ "Footprint", "Investigate", "Passive" ],- 'categories': [ "Secondary Networks" ],+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Secondary Networks"],'dataSource': {'website': "https://www.wikipedia.org/",'model': "FREE_NOAUTH_UNLIMITED",diff --git a/modules/sfp_xforce.py b/modules/sfp_xforce.pyindex 37d1a71d..00cd5711 100644--- a/modules/sfp_xforce.py+++ b/modules/sfp_xforce.py@@ -16,7 +16,7 @@ import base64from datetime import datetimeimport timefrom netaddr import IPNetwork-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_xforce(SpiderFootPlugin):"""XForce Exchange:Investigate,Passive:Reputation Systems:apikey:Obtain information from IBM X-Force Exchange"""@@ -24,9 +24,9 @@ class sfp_xforce(SpiderFootPlugin):meta = {'name': "XForce Exchange",'summary': "Obtain information from IBM X-Force Exchange",- 'flags': [ "apikey" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': ["apikey"],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://exchange.xforce.ibmcloud.com/",'model': "FREE_AUTH_LIMITED",@@ -110,8 +110,6 @@ class sfp_xforce(SpiderFootPlugin):"CO_HOSTED_SITE"]def query(self, qry, querytype):- ret = None-if querytype not in ["ipr/malware", "ipr/history", "resolve"]:querytype = "ipr/malware"@@ -176,7 +174,7 @@ class sfp_xforce(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- self.sf.error("Error processing JSON response from X-Force Exchange.", False)+ self.sf.error(f"Error processing JSON response from X-Force Exchange: {e}", False)return Nonereturn info@@ -287,7 +285,6 @@ class sfp_xforce(SpiderFootPlugin):if len(rec_history) > 0:self.sf.debug("Found history results in XForce")for result in rec_history:- reasonDescription = result.get("reasonDescription", "")created = result.get("created", None)# 2014-11-06T10:45:00.000Zif not created:@@ -316,7 +313,7 @@ class sfp_xforce(SpiderFootPlugin):self.notifyListeners(e)# ipr/malware doesn't support hostnames- if eventName in [ "CO_HOSTED_SITE", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME" ]:+ if eventName in ["CO_HOSTED_SITE", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME"]:continuerec = self.query(addr, "ipr/malware")@@ -325,7 +322,6 @@ class sfp_xforce(SpiderFootPlugin):if len(rec_malware) > 0:self.sf.debug("Found malware results in XForce")for result in rec_malware:- count = result.get("count", "")origin = result.get("origin", "")domain = result.get("domain", "")uri = result.get("uri", "")diff --git a/modules/sfp_yandexdns.py b/modules/sfp_yandexdns.pyindex 9dabc570..1aeff2d3 100644--- a/modules/sfp_yandexdns.py+++ b/modules/sfp_yandexdns.py@@ -12,7 +12,7 @@# -------------------------------------------------------------------------------import dns.resolver-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_yandexdns(SpiderFootPlugin):@@ -21,9 +21,9 @@ class sfp_yandexdns(SpiderFootPlugin):meta = {'name': "Yandex DNS",'summary': "Check if a host would be blocked by Yandex DNS",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Reputation Systems" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Reputation Systems"],'dataSource': {'website': "https://yandex.com/",'model': "FREE_NOAUTH_UNLIMITED",@@ -70,13 +70,13 @@ class sfp_yandexdns(SpiderFootPlugin):# https://dns.yandex.com/advanced/def queryAddr(self, qaddr):res = dns.resolver.Resolver()- res.nameservers = [ "77.88.8.88", "77.88.8.2" ]+ res.nameservers = ["77.88.8.88", "77.88.8.2"]try:addrs = res.query(qaddr)- self.sf.debug("Addresses returned: " + str(addrs))- except BaseException as e:- self.sf.debug("Unable to resolve " + qaddr)+ self.sf.debug(f"Addresses returned: {addrs}")+ except BaseException:+ self.sf.debug(f"Unable to resolve {qaddr}")return Falseif addrs:@@ -103,8 +103,8 @@ class sfp_yandexdns(SpiderFootPlugin):try:if self.sf.resolveHost(eventData):resolved = True- except BaseException as e:- self.sf.debug("Unable to resolve " + eventData)+ except BaseException:+ self.sf.debug(f"Unable to resolve {eventData}")return Noneif not resolved:diff --git a/modules/sfp_zoneh.py b/modules/sfp_zoneh.pyindex f0ce8f52..1733e7e1 100644--- a/modules/sfp_zoneh.py+++ b/modules/sfp_zoneh.py@@ -13,7 +13,7 @@import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+from sflib import SpiderFootPlugin, SpiderFootEventclass sfp_zoneh(SpiderFootPlugin):@@ -22,9 +22,9 @@ class sfp_zoneh(SpiderFootPlugin):meta = {'name': "Zone-H Defacement Check",'summary': "Check if a hostname/domain appears on the zone-h.org 'special defacements' RSS feed.",- 'flags': [ "" ],- 'useCases': [ "Investigate", "Passive" ],- 'categories': [ "Leaks, Dumps and Breaches" ],+ 'flags': [""],+ 'useCases': ["Investigate", "Passive"],+ 'categories': ["Leaks, Dumps and Breaches"],'dataSource': {'website': "https://zone-h.org/",'model': "FREE_NOAUTH_UNLIMITED",@@ -88,7 +88,7 @@ class sfp_zoneh(SpiderFootPlugin):"DEFACED_COHOST", "DEFACED_AFFILIATE_IPADDR"]def lookupItem(self, target, content):- grps = re.findall("<title><\!\[CDATA\[(.[^\]]*)\]\]></title>\s+<link><\!\[CDATA\[(.[^\]]*)\]\]></link>", content)+ grps = re.findall(r"<title><\!\[CDATA\[(.[^\]]*)\]\]></title>\s+<link><\!\[CDATA\[(.[^\]]*)\]\]></link>", content)for m in grps:if target in m[0]:self.sf.info("Found zoneh site: " + m[0])diff --git a/sf.py b/sf.pyindex 12a46e08..62fa5c7f 100755--- a/sf.py+++ b/sf.py@@ -139,7 +139,7 @@ if __name__ == '__main__':(addr, port) = args.l.split(":")sfConfig['__webaddr'] = addrsfConfig['__webport'] = int(port)- except BaseException as e:+ except BaseException:print("Invalid ip:port format.")sys.exit(-1)else:diff --git a/sfdb.py b/sfdb.pyindex 7e106459..16d376bc 100644--- a/sfdb.py+++ b/sfdb.py@@ -294,7 +294,7 @@ class SpiderFootDb:try:rx = re.compile(qry, re.IGNORECASE | re.DOTALL)ret = rx.match(data)- except BaseException as e:+ except BaseException:return Falsereturn ret is not None@@ -317,7 +317,7 @@ class SpiderFootDb:try:self.dbh.execute(qry)self.conn.commit()- except BaseException as e:+ except BaseException:continueself.conn.commit()# self.conn.close()