SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Tests: test_spiderfoot_module_loading: Split tests into separate functions (#1267)
bcolescfd820d96fc3
diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.pyindex 074c7b7d..df96021e 100644--- a/modules/sfp_opencorporates.py+++ b/modules/sfp_opencorporates.py@@ -22,7 +22,7 @@ class sfp_opencorporates(SpiderFootPlugin):meta = {'name': "OpenCorporates",'summary': "Look up company information from OpenCorporates.",- 'flags': [""],+ 'flags': ["apikey"],'useCases': ["Passive", "Footprint", "Investigate"],'categories': ["Search Engines"],'dataSource': {@@ -31,6 +31,14 @@ class sfp_opencorporates(SpiderFootPlugin):'references': ["https://api.opencorporates.com/documentation/API-Reference"],+ 'apiKeyInstructions': [+ "Visit https://opencorporates.com/api_accounts/new"+ "Register a new account with an email",+ "Navigate to https://opencorporates.com/users/account and select 'Get Account'",+ "Select the plan required",+ "Navigate to https://opencorporates.com/users/account",+ "The API key is listed under 'API Account'",+ ],'favIcon': "https://opencorporates.com/assets/favicons/favicon.png",'logo': "https://opencorporates.com/contents/ui/theme/img/oc-logo.svg",'description': "The largest open database of companies in the world.\n"@@ -47,7 +55,7 @@ class sfp_opencorporates(SpiderFootPlugin):optdescs = {'confidence': "Confidence that the search result objects are correct (numeric value between 0 and 100).",- 'api_key': 'OpenCorporates.com API key.'+ 'api_key': 'OpenCorporates.com API key. Without this you will be limited to 50 look-ups per day.'}results = None@@ -200,6 +208,9 @@ class sfp_opencorporates(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")+ if self.opts['api_key'] == '':+ self.sf.error("Warning: You enabled {self.__class__.__name__} but did not set an API key! Queries will be limited to 50 per day and 200 per month.")+res = self.searchCompany(f"{eventData}*")if res is None:diff --git a/test/unit/test_spiderfoot_module_loading.py b/test/unit/test_spiderfoot_module_loading.pyindex 9ad10539..93d507a4 100644--- a/test/unit/test_spiderfoot_module_loading.py+++ b/test/unit/test_spiderfoot_module_loading.py@@ -12,44 +12,142 @@ class TestSpiderFootModuleLoading(unittest.TestCase):Test SpiderFoot module loading"""- def test_module_loading(self):- sf = SpiderFoot(self.default_options)-+ @staticmethod+ def load_modules(sf):# Go through each module in the modules directory with a .py extensionsfModules = dict()mod_dir = sf.myPath() + '/modules/'for filename in os.listdir(mod_dir):- if filename.startswith("sfp_") and filename.endswith(".py"):- # Skip the module template and debugging modules- if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':- continue- modName = filename.split('.')[0]-- # Load and instantiate the module- sfModules[modName] = dict()- mod = __import__('modules.' + modName, globals(), locals(), [modName])- sfModules[modName]['object'] = getattr(mod, modName)()- sfModules[modName]['name'] = sfModules[modName]['object'].meta['name']- sfModules[modName]['cats'] = sfModules[modName]['object'].meta.get('categories', list())- sfModules[modName]['group'] = sfModules[modName]['object'].meta.get('useCases', list())- sfModules[modName]['labels'] = sfModules[modName]['object'].meta.get('flags', list())- sfModules[modName]['descr'] = sfModules[modName]['object'].meta['summary']- sfModules[modName]['provides'] = sfModules[modName]['object'].producedEvents()- sfModules[modName]['consumes'] = sfModules[modName]['object'].watchedEvents()- sfModules[modName]['meta'] = sfModules[modName]['object'].meta- if hasattr(sfModules[modName]['object'], 'opts'):- sfModules[modName]['opts'] = sfModules[modName]['object'].opts- if hasattr(sfModules[modName]['object'], 'optdescs'):- sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescs-- self.assertTrue(len(sfModules.keys()))+ if not filename.startswith("sfp_"):+ continue+ if not filename.endswith(".py"):+ continue+ # Skip the module template and debugging modules+ if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':+ continue+ modName = filename.split('.')[0]++ # Load and instantiate the module+ sfModules[modName] = dict()+ mod = __import__('modules.' + modName, globals(), locals(), [modName])+ sfModules[modName]['object'] = getattr(mod, modName)()+ sfModules[modName]['name'] = sfModules[modName]['object'].meta['name']+ sfModules[modName]['cats'] = sfModules[modName]['object'].meta.get('categories', list())+ sfModules[modName]['group'] = sfModules[modName]['object'].meta.get('useCases', list())+ sfModules[modName]['labels'] = sfModules[modName]['object'].meta.get('flags', list())+ sfModules[modName]['descr'] = sfModules[modName]['object'].meta['summary']+ sfModules[modName]['provides'] = sfModules[modName]['object'].producedEvents()+ sfModules[modName]['consumes'] = sfModules[modName]['object'].watchedEvents()+ sfModules[modName]['meta'] = sfModules[modName]['object'].meta+ if hasattr(sfModules[modName]['object'], 'opts'):+ sfModules[modName]['opts'] = sfModules[modName]['object'].opts+ if hasattr(sfModules[modName]['object'], 'optdescs'):+ sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescs++ return sfModules++ def test_module_use_cases_are_valid(self):+ sf = SpiderFoot(self.default_options)valid_use_cases = ["Footprint", "Passive", "Investigate"]++ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ if module in ["sfp__stor_db", "sfp__stor_stdout"]:+ continue++ self.assertTrue(m.get('group'))++ for group in m.get('group', list()):+ self.assertIn(group, valid_use_cases)++ def test_module_labels_are_valid(self):+ sf = SpiderFoot(self.default_options)+ valid_labels = ["", "errorprone", "tor", "slow", "invasive", "apikey", "tool"]++ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ for label in m.get('labels', list()):+ self.assertIn(label, valid_labels)++ def test_module_categories_are_valid(self):+ sf = SpiderFoot(self.default_options)valid_categories = ["Content Analysis", "Crawling and Scanning", "DNS","Leaks, Dumps and Breaches", "Passive DNS","Public Registries", "Real World", "Reputation Systems","Search Engines", "Secondary Networks", "Social Media"]+ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ self.assertIsInstance(m.get('cats'), list)+ self.assertTrue(len(m.get('cats')) <= 1)++ if module in ["sfp__stor_db", "sfp__stor_stdout"]:+ continue++ for cat in m.get('cats', list()):+ self.assertIn(cat, valid_categories)++ @unittest.skip("some modules are missing API key instructions")+ def test_modules_with_api_key_have_apiKeyInstructions(self):+ sf = SpiderFoot(self.default_options)+ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ if module in ["sfp__stor_db", "sfp__stor_stdout"]:+ continue++ self.assertTrue(m.get('meta'))+ self.assertIsInstance(m.get('meta'), dict)++ meta = m.get('meta')++ if 'dataSource' in meta and 'apikey' in m.get('labels'):+ self.assertIsInstance(meta.get('dataSource').get('apiKeyInstructions'), list)+ self.assertTrue(meta.get('dataSource').get('apiKeyInstructions'))++ def test_modules_with_api_key_options_have_apikey_label(self):+ sf = SpiderFoot(self.default_options)+ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ for opt in m.get('opts'):+ if "api_key" in opt:+ self.assertTrue("apikey" in m.get('labels'))++ def test_modules_with_invasive_flag_are_not_in_passive_use_case(self):+ sf = SpiderFoot(self.default_options)+ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ if "Passive" in m.get('group'):+ self.assertTrue("invasive" not in m.get('labels', list()))++ def test_each_module_option_has_a_description(self):+ sf = SpiderFoot(self.default_options)+ sfModules = self.load_modules(sf)+ for module in sfModules:+ m = sfModules[module]++ if module in ["sfp__stor_db", "sfp__stor_stdout"]:+ continue++ # check len(options) == len(option descriptions)+ if m.get('opts'):+ self.assertEqual(f"{module} opts: {len(m.get('opts').keys())}", f"{module} opts: {len(m.get('optdescs').keys())}")++ def test_required_module_properties_are_present_and_valid(self):+ sf = SpiderFoot(self.default_options)+ sfModules = self.load_modules(sf)for module in sfModules:m = sfModules[module]@@ -64,25 +162,9 @@ class TestSpiderFootModuleLoading(unittest.TestCase):self.assertIsInstance(m.get('consumes'), list)self.assertIsInstance(m.get('meta'), dict)- # skip debugging and output modulesif module in ["sfp__stor_db", "sfp__stor_stdout"]:continue+ self.assertTrue(m.get('cats'))self.assertTrue(m.get('group'))self.assertTrue(m.get('provides'))-- for group in m.get('group', list()):- self.assertIn(group, valid_use_cases)-- for cat in m.get('cats', list()):- self.assertIn(cat, valid_categories)-- meta = m.get('meta')-- if 'dataSource' in meta and 'apikey' in m.get('labels'):- self.assertIsInstance(meta.get('dataSource').get('apiKeyInstructions'), list)- # self.assertTrue(meta.get('dataSource').get('apiKeyInstructions'))-- # check len(options) == len(option descriptions)- if m.get('opts'):- self.assertEqual("%s opts: %s" % (module, len(m.get('opts').keys())), "%s opts: %s" % (module, len(m.get('optdescs').keys())))