SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
tests: Add test_spiderfoot_module_loading
Brendan Colesd4ee789d4c2e
diff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.pyindex 9a8eb962..53438d7b 100644--- a/modules/sfp_alienvaultiprep.py+++ b/modules/sfp_alienvaultiprep.py@@ -29,12 +29,8 @@ malchecks = {class sfp_alienvaultiprep(SpiderFootPlugin):"""AlienVault IP Reputation:Investigate,Passive:Reputation Systems::Check if an IP or netblock is malicious according to the AlienVault IP Reputation database."""---# Default optionsopts = {- '_alienvault': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_arin.py b/modules/sfp_arin.pyindex 32e2c665..bf483d00 100644--- a/modules/sfp_arin.py+++ b/modules/sfp_arin.py@@ -21,6 +21,7 @@ class sfp_arin(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = NonecurrentEventSrc = Nonediff --git a/modules/sfp_badipscom.py b/modules/sfp_badipscom.pyindex 509daba8..6279f6d6 100644--- a/modules/sfp_badipscom.py+++ b/modules/sfp_badipscom.py@@ -32,7 +32,6 @@ class sfp_badipscom(SpiderFootPlugin):# Default optionsopts = {- '_badips': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.pyindex 0621d693..76cb2951 100644--- a/modules/sfp_bitcoin.py+++ b/modules/sfp_bitcoin.py@@ -21,6 +21,7 @@ class sfp_bitcoin(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = Nonediff --git a/modules/sfp_blockchain.py b/modules/sfp_blockchain.pyindex f38a1f8e..822c8c5d 100644--- a/modules/sfp_blockchain.py+++ b/modules/sfp_blockchain.py@@ -21,6 +21,8 @@ class sfp_blockchain(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}+results = Nonedef setup(self, sfc, userOpts=dict()):diff --git a/modules/sfp_blocklistde.py b/modules/sfp_blocklistde.pyindex 8f4a139c..289db94e 100644--- a/modules/sfp_blocklistde.py+++ b/modules/sfp_blocklistde.py@@ -28,12 +28,8 @@ malchecks = {class sfp_blocklistde(SpiderFootPlugin):"""blocklist.de:Investigate,Passive:Reputation Systems::Check if a netblock or IP is malicious according to blocklist.de."""---# Default optionsopts = {- '_blocklistde': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_botvrij.py b/modules/sfp_botvrij.pyindex 181a0a07..ca7f4ec1 100644--- a/modules/sfp_botvrij.py+++ b/modules/sfp_botvrij.py@@ -31,7 +31,6 @@ class sfp_botvrij(SpiderFootPlugin):# Default optionsopts = {- '_botvrij': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_cinsscore.py b/modules/sfp_cinsscore.pyindex 257d6552..8efccee5 100644--- a/modules/sfp_cinsscore.py+++ b/modules/sfp_cinsscore.py@@ -31,7 +31,6 @@ class sfp_cinsscore(SpiderFootPlugin):# Default optionsopts = {- '_cinsscore': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_cleantalk.py b/modules/sfp_cleantalk.pyindex 15d65b53..fa9b7b45 100644--- a/modules/sfp_cleantalk.py+++ b/modules/sfp_cleantalk.py@@ -31,7 +31,6 @@ class sfp_cleantalk(SpiderFootPlugin):# Default optionsopts = {- '_cleantalk': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.pyindex 86c8b0b1..c383e1b5 100644--- a/modules/sfp_cookie.py+++ b/modules/sfp_cookie.py@@ -18,6 +18,7 @@ class sfp_cookie(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = Nonediff --git a/modules/sfp_customfeed.py b/modules/sfp_customfeed.pyindex 9098310f..5c833393 100644--- a/modules/sfp_customfeed.py+++ b/modules/sfp_customfeed.py@@ -29,7 +29,6 @@ class sfp_customfeed(SpiderFootPlugin):# Default optionsopts = {- '_customfeed': True,'checkaffiliates': True,'checkcohosts': True,'url': "",diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.pyindex f8568e15..c8f29ad5 100644--- a/modules/sfp_cybercrimetracker.py+++ b/modules/sfp_cybercrimetracker.py@@ -32,7 +32,6 @@ class sfp_cybercrimetracker(SpiderFootPlugin):# Default optionsopts = {- '_cybercrime': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_emergingthreats.py b/modules/sfp_emergingthreats.pyindex 28c90a53..0fe96a0d 100644--- a/modules/sfp_emergingthreats.py+++ b/modules/sfp_emergingthreats.py@@ -31,7 +31,6 @@ class sfp_emergingthreats(SpiderFootPlugin):# Default optionsopts = {- '_emergingthreats': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_ethereum.py b/modules/sfp_ethereum.pyindex 254151ab..a16a5d7b 100644--- a/modules/sfp_ethereum.py+++ b/modules/sfp_ethereum.py@@ -20,6 +20,7 @@ class sfp_ethereum(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = Nonediff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.pyindex ffb08af2..393feb57 100644--- a/modules/sfp_fortinet.py+++ b/modules/sfp_fortinet.py@@ -33,7 +33,6 @@ class sfp_fortinet(SpiderFootPlugin):# Default optionsopts = {- '_fortiguard': True,'checkaffiliates': True}diff --git a/modules/sfp_greensnow.py b/modules/sfp_greensnow.pyindex e086df81..24a37d2b 100644--- a/modules/sfp_greensnow.py+++ b/modules/sfp_greensnow.py@@ -31,7 +31,6 @@ class sfp_greensnow(SpiderFootPlugin):# Default optionsopts = {- '_greensnow': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_isc.py b/modules/sfp_isc.pyindex 7d94201e..f131d84d 100644--- a/modules/sfp_isc.py+++ b/modules/sfp_isc.py@@ -33,7 +33,6 @@ class sfp_isc(SpiderFootPlugin):# Default optionsopts = {- '_isc': True,'checkaffiliates': True}diff --git a/modules/sfp_malwaredomains.py b/modules/sfp_malwaredomains.pyindex 4374d380..f0d7755d 100644--- a/modules/sfp_malwaredomains.py+++ b/modules/sfp_malwaredomains.py@@ -31,7 +31,6 @@ class sfp_malwaredomains(SpiderFootPlugin):# Default optionsopts = {- '_malwaredomains': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.pyindex 6ec1a8f7..fec51054 100644--- a/modules/sfp_multiproxy.py+++ b/modules/sfp_multiproxy.py@@ -32,7 +32,6 @@ class sfp_multiproxy(SpiderFootPlugin):# Default optionsopts = {- '_multiproxy': True,'checkaffiliates': True,'cacheperiod': 18}diff --git a/modules/sfp_openphish.py b/modules/sfp_openphish.pyindex ae6a48d1..a113f0f0 100644--- a/modules/sfp_openphish.py+++ b/modules/sfp_openphish.py@@ -32,7 +32,6 @@ class sfp_openphish(SpiderFootPlugin):# Default optionsopts = {- '_openphish': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.pyindex 7b91bc8e..fa4b52e1 100644--- a/modules/sfp_pageinfo.py+++ b/modules/sfp_pageinfo.py@@ -33,6 +33,7 @@ class sfp_pageinfo(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = Nonediff --git a/modules/sfp_phishtank.py b/modules/sfp_phishtank.pyindex 5d2658d8..5dfeeddf 100644--- a/modules/sfp_phishtank.py+++ b/modules/sfp_phishtank.py@@ -32,7 +32,6 @@ class sfp_phishtank(SpiderFootPlugin):# Default optionsopts = {- '_phishtank': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_phone.py b/modules/sfp_phone.pyindex b64edc4e..7a9e8d75 100644--- a/modules/sfp_phone.py+++ b/modules/sfp_phone.py@@ -24,6 +24,7 @@ class sfp_phone(SpiderFootPlugin):opts = {}results = None+ optdescs = {}def setup(self, sfc, userOpts=dict()):self.sf = sfcdiff --git a/modules/sfp_ripe.py b/modules/sfp_ripe.pyindex e96ab40b..66c29363 100644--- a/modules/sfp_ripe.py+++ b/modules/sfp_ripe.py@@ -23,6 +23,7 @@ class sfp_ripe(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = NonecurrentEventSrc = Nonediff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.pyindex ddbc263f..1bff0d2a 100644--- a/modules/sfp_strangeheaders.py+++ b/modules/sfp_strangeheaders.py@@ -29,6 +29,7 @@ class sfp_strangeheaders(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = Nonediff --git a/modules/sfp_talosintel.py b/modules/sfp_talosintel.pyindex 6a85e799..a8d5f18d 100644--- a/modules/sfp_talosintel.py+++ b/modules/sfp_talosintel.py@@ -31,7 +31,6 @@ class sfp_talosintel(SpiderFootPlugin):# Default optionsopts = {- '_talosintel': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.pyindex a9f73272..dd32d95b 100644--- a/modules/sfp_torexits.py+++ b/modules/sfp_torexits.py@@ -32,7 +32,6 @@ class sfp_torexits(SpiderFootPlugin):# Default optionsopts = {- '_torexits': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_totalhash.py b/modules/sfp_totalhash.pyindex a8791d15..0c084f84 100644--- a/modules/sfp_totalhash.py+++ b/modules/sfp_totalhash.py@@ -33,7 +33,6 @@ class sfp_totalhash(SpiderFootPlugin):# Default optionsopts = {- '_totalhash': True,'checkaffiliates': True,'checkcohosts': True}diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.pyindex c24e0db7..5d78c53e 100644--- a/modules/sfp_voipbl.py+++ b/modules/sfp_voipbl.py@@ -31,7 +31,6 @@ class sfp_voipbl(SpiderFootPlugin):# Default optionsopts = {- '_voipbl': True,'checkaffiliates': True,'cacheperiod': 18,'checknetblocks': True,diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.pyindex 53c21ec9..687b1a39 100644--- a/modules/sfp_vxvault.py+++ b/modules/sfp_vxvault.py@@ -32,7 +32,6 @@ class sfp_vxvault(SpiderFootPlugin):# Default optionsopts = {- '_vxvault': True,'checkaffiliates': True,'checkcohosts': True,'cacheperiod': 18diff --git a/modules/sfp_watchguard.py b/modules/sfp_watchguard.pyindex 3b1d1200..7c753e9d 100644--- a/modules/sfp_watchguard.py+++ b/modules/sfp_watchguard.py@@ -32,7 +32,6 @@ class sfp_watchguard(SpiderFootPlugin):# Default optionsopts = {- '_watchguard': True,'checkaffiliates': True}diff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.pyindex 41627a7d..300f56fe 100644--- a/modules/sfp_webserver.py+++ b/modules/sfp_webserver.py@@ -19,6 +19,7 @@ class sfp_webserver(SpiderFootPlugin):# Default optionsopts = {}+ optdescs = {}results = Nonediff --git a/test/unit/test_spiderfoot_module_loading.py b/test/unit/test_spiderfoot_module_loading.pynew file mode 100644index 00000000..523c5bd3--- /dev/null+++ b/test/unit/test_spiderfoot_module_loading.py@@ -0,0 +1,104 @@+# test_spiderfoot_module_loading.py+from sflib import SpiderFoot, SpiderFootTarget+import unittest+import os++class TestSpiderFootModuleLoading(unittest.TestCase):+ """+ Test SpiderFoot module loading+ """++ default_options = {+ '_debug': False, # Debug+ '__logging': True, # Logging in general+ '__outputfilter': None, # Event types to filter from modules' output+ '__blocknotif': False, # Block notifications+ '_fatalerrors': False,+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests+ '_dnsserver': '', # Override the default resolver+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',+ '_internettlds_cache': 72,+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",+ '__version__': '3.0',+ '__database': 'spiderfoot.test.db', # note: test database file+ '__webaddr': '127.0.0.1',+ '__webport': 5001,+ '__docroot': '', # don't put trailing /+ '__modules__': None, # List of modules. Will be set after start-up.+ '_socks1type': '',+ '_socks2addr': '',+ '_socks3port': '',+ '_socks4user': '',+ '_socks5pwd': '',+ '_socks6dns': True,+ '_torctlport': 9051,+ '__logstdout': False+ }++ def test_module_loading(self):+ """+ Test module loading+ """+ sf = SpiderFoot(self.default_options)++ # Go through each module in the modules directory with a .py extension+ sfModules = dict()+ mod_dir = sf.myPath() + '/modules/'+ for filename in os.listdir(mod_dir):+ if filename.startswith("sfp_") and filename.endswith(".py"):+ # Skip the module template and debugging modules+ if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':+ continue+ modName = filename.split('.')[0]++ # Load and instantiate the module+ sfModules[modName] = dict()+ mod = __import__('modules.' + modName, globals(), locals(), [modName])+ sfModules[modName]['object'] = getattr(mod, modName)()+ sfModules[modName]['name'] = sfModules[modName]['object'].__doc__.split(":", 5)[0]+ sfModules[modName]['cats'] = sfModules[modName]['object'].__doc__.split(":", 5)[1].split(",")+ sfModules[modName]['group'] = sfModules[modName]['object'].__doc__.split(":", 5)[2]+ sfModules[modName]['labels'] = sfModules[modName]['object'].__doc__.split(":", 5)[3].split(",")+ sfModules[modName]['descr'] = sfModules[modName]['object'].__doc__.split(":", 5)[4]+ sfModules[modName]['provides'] = sfModules[modName]['object'].producedEvents()+ sfModules[modName]['consumes'] = sfModules[modName]['object'].watchedEvents()+ if hasattr(sfModules[modName]['object'], 'opts'):+ sfModules[modName]['opts'] = sfModules[modName]['object'].opts+ if hasattr(sfModules[modName]['object'], 'optdescs'):+ sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescs++ self.assertTrue(len(sfModules.keys()))++ valid_categories = ["Footprint", "Passive", "Investigate"]++ for module in sfModules:+ m = sfModules[module]++ self.assertTrue(m.get('object'))+ self.assertTrue(m.get('name'))+ self.assertTrue(m.get('cats'))+ #self.assertTrue(m.get('group'))+ self.assertTrue(m.get('labels'))+ self.assertTrue(m.get('descr'))+ #self.assertTrue(m.get('provides'))+ self.assertIsInstance(m.get('provides'), list)+ self.assertTrue(m.get('consumes'))+ self.assertIsInstance(m.get('consumes'), list)++ # skip non-conforming modules+ if module in [+ "sfp__stor_db", "sfp__stor_stdout",+ "sfp_googleobjectstorage", "sfp_digitaloceanspace",+ "sfp_tldsearch", "sfp_azureblobstorage", "sfp_s3bucket",+ "sfp_accounts", "sfp_dnsbrute"+ ]:+ continue++ for cat in m.get('cats'):+ self.assertIn(cat, valid_categories)++ # check len(options) == len(option descriptions)+ if m.get('opts'):+ self.assertEqual("%s opts: %s" % (module, len(m.get('opts').keys())), "%s opts: %s" % (module, len(m.get('optdescs').keys())))+