Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

tests: Add test_spiderfoot_module_loading

Brendan Colesd4ee789d4c2e
diff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.py
index 9a8eb962..53438d7b 100644
--- a/modules/sfp_alienvaultiprep.py
+++ b/modules/sfp_alienvaultiprep.py
@@ -29,12 +29,8 @@ malchecks = {
class sfp_alienvaultiprep(SpiderFootPlugin):
"""AlienVault IP Reputation:Investigate,Passive:Reputation Systems::Check if an IP or netblock is malicious according to the AlienVault IP Reputation database."""
-
-
-
# Default options
opts = {
- '_alienvault': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_arin.py b/modules/sfp_arin.py
index 32e2c665..bf483d00 100644
--- a/modules/sfp_arin.py
+++ b/modules/sfp_arin.py
@@ -21,6 +21,7 @@ class sfp_arin(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
currentEventSrc = None
diff --git a/modules/sfp_badipscom.py b/modules/sfp_badipscom.py
index 509daba8..6279f6d6 100644
--- a/modules/sfp_badipscom.py
+++ b/modules/sfp_badipscom.py
@@ -32,7 +32,6 @@ class sfp_badipscom(SpiderFootPlugin):
# Default options
opts = {
- '_badips': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.py
index 0621d693..76cb2951 100644
--- a/modules/sfp_bitcoin.py
+++ b/modules/sfp_bitcoin.py
@@ -21,6 +21,7 @@ class sfp_bitcoin(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
diff --git a/modules/sfp_blockchain.py b/modules/sfp_blockchain.py
index f38a1f8e..822c8c5d 100644
--- a/modules/sfp_blockchain.py
+++ b/modules/sfp_blockchain.py
@@ -21,6 +21,8 @@ class sfp_blockchain(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
+
results = None
def setup(self, sfc, userOpts=dict()):
diff --git a/modules/sfp_blocklistde.py b/modules/sfp_blocklistde.py
index 8f4a139c..289db94e 100644
--- a/modules/sfp_blocklistde.py
+++ b/modules/sfp_blocklistde.py
@@ -28,12 +28,8 @@ malchecks = {
class sfp_blocklistde(SpiderFootPlugin):
"""blocklist.de:Investigate,Passive:Reputation Systems::Check if a netblock or IP is malicious according to blocklist.de."""
-
-
-
# Default options
opts = {
- '_blocklistde': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_botvrij.py b/modules/sfp_botvrij.py
index 181a0a07..ca7f4ec1 100644
--- a/modules/sfp_botvrij.py
+++ b/modules/sfp_botvrij.py
@@ -31,7 +31,6 @@ class sfp_botvrij(SpiderFootPlugin):
# Default options
opts = {
- '_botvrij': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_cinsscore.py b/modules/sfp_cinsscore.py
index 257d6552..8efccee5 100644
--- a/modules/sfp_cinsscore.py
+++ b/modules/sfp_cinsscore.py
@@ -31,7 +31,6 @@ class sfp_cinsscore(SpiderFootPlugin):
# Default options
opts = {
- '_cinsscore': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_cleantalk.py b/modules/sfp_cleantalk.py
index 15d65b53..fa9b7b45 100644
--- a/modules/sfp_cleantalk.py
+++ b/modules/sfp_cleantalk.py
@@ -31,7 +31,6 @@ class sfp_cleantalk(SpiderFootPlugin):
# Default options
opts = {
- '_cleantalk': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.py
index 86c8b0b1..c383e1b5 100644
--- a/modules/sfp_cookie.py
+++ b/modules/sfp_cookie.py
@@ -18,6 +18,7 @@ class sfp_cookie(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
diff --git a/modules/sfp_customfeed.py b/modules/sfp_customfeed.py
index 9098310f..5c833393 100644
--- a/modules/sfp_customfeed.py
+++ b/modules/sfp_customfeed.py
@@ -29,7 +29,6 @@ class sfp_customfeed(SpiderFootPlugin):
# Default options
opts = {
- '_customfeed': True,
'checkaffiliates': True,
'checkcohosts': True,
'url': "",
diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.py
index f8568e15..c8f29ad5 100644
--- a/modules/sfp_cybercrimetracker.py
+++ b/modules/sfp_cybercrimetracker.py
@@ -32,7 +32,6 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
# Default options
opts = {
- '_cybercrime': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_emergingthreats.py b/modules/sfp_emergingthreats.py
index 28c90a53..0fe96a0d 100644
--- a/modules/sfp_emergingthreats.py
+++ b/modules/sfp_emergingthreats.py
@@ -31,7 +31,6 @@ class sfp_emergingthreats(SpiderFootPlugin):
# Default options
opts = {
- '_emergingthreats': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_ethereum.py b/modules/sfp_ethereum.py
index 254151ab..a16a5d7b 100644
--- a/modules/sfp_ethereum.py
+++ b/modules/sfp_ethereum.py
@@ -20,6 +20,7 @@ class sfp_ethereum(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
diff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.py
index ffb08af2..393feb57 100644
--- a/modules/sfp_fortinet.py
+++ b/modules/sfp_fortinet.py
@@ -33,7 +33,6 @@ class sfp_fortinet(SpiderFootPlugin):
# Default options
opts = {
- '_fortiguard': True,
'checkaffiliates': True
}
diff --git a/modules/sfp_greensnow.py b/modules/sfp_greensnow.py
index e086df81..24a37d2b 100644
--- a/modules/sfp_greensnow.py
+++ b/modules/sfp_greensnow.py
@@ -31,7 +31,6 @@ class sfp_greensnow(SpiderFootPlugin):
# Default options
opts = {
- '_greensnow': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_isc.py b/modules/sfp_isc.py
index 7d94201e..f131d84d 100644
--- a/modules/sfp_isc.py
+++ b/modules/sfp_isc.py
@@ -33,7 +33,6 @@ class sfp_isc(SpiderFootPlugin):
# Default options
opts = {
- '_isc': True,
'checkaffiliates': True
}
diff --git a/modules/sfp_malwaredomains.py b/modules/sfp_malwaredomains.py
index 4374d380..f0d7755d 100644
--- a/modules/sfp_malwaredomains.py
+++ b/modules/sfp_malwaredomains.py
@@ -31,7 +31,6 @@ class sfp_malwaredomains(SpiderFootPlugin):
# Default options
opts = {
- '_malwaredomains': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.py
index 6ec1a8f7..fec51054 100644
--- a/modules/sfp_multiproxy.py
+++ b/modules/sfp_multiproxy.py
@@ -32,7 +32,6 @@ class sfp_multiproxy(SpiderFootPlugin):
# Default options
opts = {
- '_multiproxy': True,
'checkaffiliates': True,
'cacheperiod': 18
}
diff --git a/modules/sfp_openphish.py b/modules/sfp_openphish.py
index ae6a48d1..a113f0f0 100644
--- a/modules/sfp_openphish.py
+++ b/modules/sfp_openphish.py
@@ -32,7 +32,6 @@ class sfp_openphish(SpiderFootPlugin):
# Default options
opts = {
- '_openphish': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.py
index 7b91bc8e..fa4b52e1 100644
--- a/modules/sfp_pageinfo.py
+++ b/modules/sfp_pageinfo.py
@@ -33,6 +33,7 @@ class sfp_pageinfo(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
diff --git a/modules/sfp_phishtank.py b/modules/sfp_phishtank.py
index 5d2658d8..5dfeeddf 100644
--- a/modules/sfp_phishtank.py
+++ b/modules/sfp_phishtank.py
@@ -32,7 +32,6 @@ class sfp_phishtank(SpiderFootPlugin):
# Default options
opts = {
- '_phishtank': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_phone.py b/modules/sfp_phone.py
index b64edc4e..7a9e8d75 100644
--- a/modules/sfp_phone.py
+++ b/modules/sfp_phone.py
@@ -24,6 +24,7 @@ class sfp_phone(SpiderFootPlugin):
opts = {}
results = None
+ optdescs = {}
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
diff --git a/modules/sfp_ripe.py b/modules/sfp_ripe.py
index e96ab40b..66c29363 100644
--- a/modules/sfp_ripe.py
+++ b/modules/sfp_ripe.py
@@ -23,6 +23,7 @@ class sfp_ripe(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
currentEventSrc = None
diff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.py
index ddbc263f..1bff0d2a 100644
--- a/modules/sfp_strangeheaders.py
+++ b/modules/sfp_strangeheaders.py
@@ -29,6 +29,7 @@ class sfp_strangeheaders(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
diff --git a/modules/sfp_talosintel.py b/modules/sfp_talosintel.py
index 6a85e799..a8d5f18d 100644
--- a/modules/sfp_talosintel.py
+++ b/modules/sfp_talosintel.py
@@ -31,7 +31,6 @@ class sfp_talosintel(SpiderFootPlugin):
# Default options
opts = {
- '_talosintel': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.py
index a9f73272..dd32d95b 100644
--- a/modules/sfp_torexits.py
+++ b/modules/sfp_torexits.py
@@ -32,7 +32,6 @@ class sfp_torexits(SpiderFootPlugin):
# Default options
opts = {
- '_torexits': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_totalhash.py b/modules/sfp_totalhash.py
index a8791d15..0c084f84 100644
--- a/modules/sfp_totalhash.py
+++ b/modules/sfp_totalhash.py
@@ -33,7 +33,6 @@ class sfp_totalhash(SpiderFootPlugin):
# Default options
opts = {
- '_totalhash': True,
'checkaffiliates': True,
'checkcohosts': True
}
diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.py
index c24e0db7..5d78c53e 100644
--- a/modules/sfp_voipbl.py
+++ b/modules/sfp_voipbl.py
@@ -31,7 +31,6 @@ class sfp_voipbl(SpiderFootPlugin):
# Default options
opts = {
- '_voipbl': True,
'checkaffiliates': True,
'cacheperiod': 18,
'checknetblocks': True,
diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.py
index 53c21ec9..687b1a39 100644
--- a/modules/sfp_vxvault.py
+++ b/modules/sfp_vxvault.py
@@ -32,7 +32,6 @@ class sfp_vxvault(SpiderFootPlugin):
# Default options
opts = {
- '_vxvault': True,
'checkaffiliates': True,
'checkcohosts': True,
'cacheperiod': 18
diff --git a/modules/sfp_watchguard.py b/modules/sfp_watchguard.py
index 3b1d1200..7c753e9d 100644
--- a/modules/sfp_watchguard.py
+++ b/modules/sfp_watchguard.py
@@ -32,7 +32,6 @@ class sfp_watchguard(SpiderFootPlugin):
# Default options
opts = {
- '_watchguard': True,
'checkaffiliates': True
}
diff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.py
index 41627a7d..300f56fe 100644
--- a/modules/sfp_webserver.py
+++ b/modules/sfp_webserver.py
@@ -19,6 +19,7 @@ class sfp_webserver(SpiderFootPlugin):
# Default options
opts = {}
+ optdescs = {}
results = None
diff --git a/test/unit/test_spiderfoot_module_loading.py b/test/unit/test_spiderfoot_module_loading.py
new file mode 100644
index 00000000..523c5bd3
--- /dev/null
+++ b/test/unit/test_spiderfoot_module_loading.py
@@ -0,0 +1,104 @@
+# test_spiderfoot_module_loading.py
+from sflib import SpiderFoot, SpiderFootTarget
+import unittest
+import os
+
+class TestSpiderFootModuleLoading(unittest.TestCase):
+ """
+ Test SpiderFoot module loading
+ """
+
+ default_options = {
+ '_debug': False, # Debug
+ '__logging': True, # Logging in general
+ '__outputfilter': None, # Event types to filter from modules' output
+ '__blocknotif': False, # Block notifications
+ '_fatalerrors': False,
+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
+ '_dnsserver': '', # Override the default resolver
+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch
+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
+ '_internettlds_cache': 72,
+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
+ '__version__': '3.0',
+ '__database': 'spiderfoot.test.db', # note: test database file
+ '__webaddr': '127.0.0.1',
+ '__webport': 5001,
+ '__docroot': '', # don't put trailing /
+ '__modules__': None, # List of modules. Will be set after start-up.
+ '_socks1type': '',
+ '_socks2addr': '',
+ '_socks3port': '',
+ '_socks4user': '',
+ '_socks5pwd': '',
+ '_socks6dns': True,
+ '_torctlport': 9051,
+ '__logstdout': False
+ }
+
+ def test_module_loading(self):
+ """
+ Test module loading
+ """
+ sf = SpiderFoot(self.default_options)
+
+ # Go through each module in the modules directory with a .py extension
+ sfModules = dict()
+ mod_dir = sf.myPath() + '/modules/'
+ for filename in os.listdir(mod_dir):
+ if filename.startswith("sfp_") and filename.endswith(".py"):
+ # Skip the module template and debugging modules
+ if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':
+ continue
+ modName = filename.split('.')[0]
+
+ # Load and instantiate the module
+ sfModules[modName] = dict()
+ mod = __import__('modules.' + modName, globals(), locals(), [modName])
+ sfModules[modName]['object'] = getattr(mod, modName)()
+ sfModules[modName]['name'] = sfModules[modName]['object'].__doc__.split(":", 5)[0]
+ sfModules[modName]['cats'] = sfModules[modName]['object'].__doc__.split(":", 5)[1].split(",")
+ sfModules[modName]['group'] = sfModules[modName]['object'].__doc__.split(":", 5)[2]
+ sfModules[modName]['labels'] = sfModules[modName]['object'].__doc__.split(":", 5)[3].split(",")
+ sfModules[modName]['descr'] = sfModules[modName]['object'].__doc__.split(":", 5)[4]
+ sfModules[modName]['provides'] = sfModules[modName]['object'].producedEvents()
+ sfModules[modName]['consumes'] = sfModules[modName]['object'].watchedEvents()
+ if hasattr(sfModules[modName]['object'], 'opts'):
+ sfModules[modName]['opts'] = sfModules[modName]['object'].opts
+ if hasattr(sfModules[modName]['object'], 'optdescs'):
+ sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescs
+
+ self.assertTrue(len(sfModules.keys()))
+
+ valid_categories = ["Footprint", "Passive", "Investigate"]
+
+ for module in sfModules:
+ m = sfModules[module]
+
+ self.assertTrue(m.get('object'))
+ self.assertTrue(m.get('name'))
+ self.assertTrue(m.get('cats'))
+ #self.assertTrue(m.get('group'))
+ self.assertTrue(m.get('labels'))
+ self.assertTrue(m.get('descr'))
+ #self.assertTrue(m.get('provides'))
+ self.assertIsInstance(m.get('provides'), list)
+ self.assertTrue(m.get('consumes'))
+ self.assertIsInstance(m.get('consumes'), list)
+
+ # skip non-conforming modules
+ if module in [
+ "sfp__stor_db", "sfp__stor_stdout",
+ "sfp_googleobjectstorage", "sfp_digitaloceanspace",
+ "sfp_tldsearch", "sfp_azureblobstorage", "sfp_s3bucket",
+ "sfp_accounts", "sfp_dnsbrute"
+ ]:
+ continue
+
+ for cat in m.get('cats'):
+ self.assertIn(cat, valid_categories)
+
+ # check len(options) == len(option descriptions)
+ if m.get('opts'):
+ self.assertEqual("%s opts: %s" % (module, len(m.get('opts').keys())), "%s opts: %s" % (module, len(m.get('optdescs').keys())))
+