Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Tests: test_spiderfoot_module_loading: Split tests into separate functions (#1267)

bcolescfd820d96fc3
diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.py
index 074c7b7d..df96021e 100644
--- a/modules/sfp_opencorporates.py
+++ b/modules/sfp_opencorporates.py
@@ -22,7 +22,7 @@ class sfp_opencorporates(SpiderFootPlugin):
meta = {
'name': "OpenCorporates",
'summary': "Look up company information from OpenCorporates.",
- 'flags': [""],
+ 'flags': ["apikey"],
'useCases': ["Passive", "Footprint", "Investigate"],
'categories': ["Search Engines"],
'dataSource': {
@@ -31,6 +31,14 @@ class sfp_opencorporates(SpiderFootPlugin):
'references': [
"https://api.opencorporates.com/documentation/API-Reference"
],
+ 'apiKeyInstructions': [
+ "Visit https://opencorporates.com/api_accounts/new"
+ "Register a new account with an email",
+ "Navigate to https://opencorporates.com/users/account and select 'Get Account'",
+ "Select the plan required",
+ "Navigate to https://opencorporates.com/users/account",
+ "The API key is listed under 'API Account'",
+ ],
'favIcon': "https://opencorporates.com/assets/favicons/favicon.png",
'logo': "https://opencorporates.com/contents/ui/theme/img/oc-logo.svg",
'description': "The largest open database of companies in the world.\n"
@@ -47,7 +55,7 @@ class sfp_opencorporates(SpiderFootPlugin):
optdescs = {
'confidence': "Confidence that the search result objects are correct (numeric value between 0 and 100).",
- 'api_key': 'OpenCorporates.com API key.'
+ 'api_key': 'OpenCorporates.com API key. Without this you will be limited to 50 look-ups per day.'
}
results = None
@@ -200,6 +208,9 @@ class sfp_opencorporates(SpiderFootPlugin):
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
+ if self.opts['api_key'] == '':
+ self.sf.error("Warning: You enabled {self.__class__.__name__} but did not set an API key! Queries will be limited to 50 per day and 200 per month.")
+
res = self.searchCompany(f"{eventData}*")
if res is None:
diff --git a/test/unit/test_spiderfoot_module_loading.py b/test/unit/test_spiderfoot_module_loading.py
index 9ad10539..93d507a4 100644
--- a/test/unit/test_spiderfoot_module_loading.py
+++ b/test/unit/test_spiderfoot_module_loading.py
@@ -12,44 +12,142 @@ class TestSpiderFootModuleLoading(unittest.TestCase):
Test SpiderFoot module loading
"""
- def test_module_loading(self):
- sf = SpiderFoot(self.default_options)
-
+ @staticmethod
+ def load_modules(sf):
# Go through each module in the modules directory with a .py extension
sfModules = dict()
mod_dir = sf.myPath() + '/modules/'
for filename in os.listdir(mod_dir):
- if filename.startswith("sfp_") and filename.endswith(".py"):
- # Skip the module template and debugging modules
- if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':
- continue
- modName = filename.split('.')[0]
-
- # Load and instantiate the module
- sfModules[modName] = dict()
- mod = __import__('modules.' + modName, globals(), locals(), [modName])
- sfModules[modName]['object'] = getattr(mod, modName)()
- sfModules[modName]['name'] = sfModules[modName]['object'].meta['name']
- sfModules[modName]['cats'] = sfModules[modName]['object'].meta.get('categories', list())
- sfModules[modName]['group'] = sfModules[modName]['object'].meta.get('useCases', list())
- sfModules[modName]['labels'] = sfModules[modName]['object'].meta.get('flags', list())
- sfModules[modName]['descr'] = sfModules[modName]['object'].meta['summary']
- sfModules[modName]['provides'] = sfModules[modName]['object'].producedEvents()
- sfModules[modName]['consumes'] = sfModules[modName]['object'].watchedEvents()
- sfModules[modName]['meta'] = sfModules[modName]['object'].meta
- if hasattr(sfModules[modName]['object'], 'opts'):
- sfModules[modName]['opts'] = sfModules[modName]['object'].opts
- if hasattr(sfModules[modName]['object'], 'optdescs'):
- sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescs
-
- self.assertTrue(len(sfModules.keys()))
+ if not filename.startswith("sfp_"):
+ continue
+ if not filename.endswith(".py"):
+ continue
+ # Skip the module template and debugging modules
+ if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':
+ continue
+ modName = filename.split('.')[0]
+
+ # Load and instantiate the module
+ sfModules[modName] = dict()
+ mod = __import__('modules.' + modName, globals(), locals(), [modName])
+ sfModules[modName]['object'] = getattr(mod, modName)()
+ sfModules[modName]['name'] = sfModules[modName]['object'].meta['name']
+ sfModules[modName]['cats'] = sfModules[modName]['object'].meta.get('categories', list())
+ sfModules[modName]['group'] = sfModules[modName]['object'].meta.get('useCases', list())
+ sfModules[modName]['labels'] = sfModules[modName]['object'].meta.get('flags', list())
+ sfModules[modName]['descr'] = sfModules[modName]['object'].meta['summary']
+ sfModules[modName]['provides'] = sfModules[modName]['object'].producedEvents()
+ sfModules[modName]['consumes'] = sfModules[modName]['object'].watchedEvents()
+ sfModules[modName]['meta'] = sfModules[modName]['object'].meta
+ if hasattr(sfModules[modName]['object'], 'opts'):
+ sfModules[modName]['opts'] = sfModules[modName]['object'].opts
+ if hasattr(sfModules[modName]['object'], 'optdescs'):
+ sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescs
+
+ return sfModules
+
+ def test_module_use_cases_are_valid(self):
+ sf = SpiderFoot(self.default_options)
valid_use_cases = ["Footprint", "Passive", "Investigate"]
+
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ if module in ["sfp__stor_db", "sfp__stor_stdout"]:
+ continue
+
+ self.assertTrue(m.get('group'))
+
+ for group in m.get('group', list()):
+ self.assertIn(group, valid_use_cases)
+
+ def test_module_labels_are_valid(self):
+ sf = SpiderFoot(self.default_options)
+ valid_labels = ["", "errorprone", "tor", "slow", "invasive", "apikey", "tool"]
+
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ for label in m.get('labels', list()):
+ self.assertIn(label, valid_labels)
+
+ def test_module_categories_are_valid(self):
+ sf = SpiderFoot(self.default_options)
valid_categories = ["Content Analysis", "Crawling and Scanning", "DNS",
"Leaks, Dumps and Breaches", "Passive DNS",
"Public Registries", "Real World", "Reputation Systems",
"Search Engines", "Secondary Networks", "Social Media"]
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ self.assertIsInstance(m.get('cats'), list)
+ self.assertTrue(len(m.get('cats')) <= 1)
+
+ if module in ["sfp__stor_db", "sfp__stor_stdout"]:
+ continue
+
+ for cat in m.get('cats', list()):
+ self.assertIn(cat, valid_categories)
+
+ @unittest.skip("some modules are missing API key instructions")
+ def test_modules_with_api_key_have_apiKeyInstructions(self):
+ sf = SpiderFoot(self.default_options)
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ if module in ["sfp__stor_db", "sfp__stor_stdout"]:
+ continue
+
+ self.assertTrue(m.get('meta'))
+ self.assertIsInstance(m.get('meta'), dict)
+
+ meta = m.get('meta')
+
+ if 'dataSource' in meta and 'apikey' in m.get('labels'):
+ self.assertIsInstance(meta.get('dataSource').get('apiKeyInstructions'), list)
+ self.assertTrue(meta.get('dataSource').get('apiKeyInstructions'))
+
+ def test_modules_with_api_key_options_have_apikey_label(self):
+ sf = SpiderFoot(self.default_options)
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ for opt in m.get('opts'):
+ if "api_key" in opt:
+ self.assertTrue("apikey" in m.get('labels'))
+
+ def test_modules_with_invasive_flag_are_not_in_passive_use_case(self):
+ sf = SpiderFoot(self.default_options)
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ if "Passive" in m.get('group'):
+ self.assertTrue("invasive" not in m.get('labels', list()))
+
+ def test_each_module_option_has_a_description(self):
+ sf = SpiderFoot(self.default_options)
+ sfModules = self.load_modules(sf)
+ for module in sfModules:
+ m = sfModules[module]
+
+ if module in ["sfp__stor_db", "sfp__stor_stdout"]:
+ continue
+
+ # check len(options) == len(option descriptions)
+ if m.get('opts'):
+ self.assertEqual(f"{module} opts: {len(m.get('opts').keys())}", f"{module} opts: {len(m.get('optdescs').keys())}")
+
+ def test_required_module_properties_are_present_and_valid(self):
+ sf = SpiderFoot(self.default_options)
+ sfModules = self.load_modules(sf)
for module in sfModules:
m = sfModules[module]
@@ -64,25 +162,9 @@ class TestSpiderFootModuleLoading(unittest.TestCase):
self.assertIsInstance(m.get('consumes'), list)
self.assertIsInstance(m.get('meta'), dict)
- # skip debugging and output modules
if module in ["sfp__stor_db", "sfp__stor_stdout"]:
continue
+ self.assertTrue(m.get('cats'))
self.assertTrue(m.get('group'))
self.assertTrue(m.get('provides'))
-
- for group in m.get('group', list()):
- self.assertIn(group, valid_use_cases)
-
- for cat in m.get('cats', list()):
- self.assertIn(cat, valid_categories)
-
- meta = m.get('meta')
-
- if 'dataSource' in meta and 'apikey' in m.get('labels'):
- self.assertIsInstance(meta.get('dataSource').get('apiKeyInstructions'), list)
- # self.assertTrue(meta.get('dataSource').get('apiKeyInstructions'))
-
- # check len(options) == len(option descriptions)
- if m.get('opts'):
- self.assertEqual("%s opts: %s" % (module, len(m.get('opts').keys())), "%s opts: %s" % (module, len(m.get('optdescs').keys())))