SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Add sfp_certspotter (#1268)
bcolesffc47770cf4b
diff --git a/modules/sfp_certspotter.py b/modules/sfp_certspotter.pynew file mode 100644index 00000000..fa5af1fe--- /dev/null+++ b/modules/sfp_certspotter.py@@ -0,0 +1,271 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_certspotter+# Purpose: Gather information about SSL certificates from SSLMate CertSpotter API.+#+# Author: <[email protected]>+#+# Created: 2021-08-15+# Copyright: (c) bcoles+# Licence: GPL+# -------------------------------------------------------------------------------++import base64+import json+import time+import urllib.error+import urllib.parse+import urllib.request++from spiderfoot import SpiderFootEvent, SpiderFootPlugin+++class sfp_certspotter(SpiderFootPlugin):+ meta = {+ 'name': "CertSpotter",+ 'summary': "Gather information about SSL certificates from SSLMate CertSpotter API.",+ 'flags': ["apikey"],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Crawling and Scanning"],+ 'dataSource': {+ 'website': "https://sslmate.com/certspotter/",+ 'model': "FREE_AUTH_LIMITED",+ 'references': [+ "https://sslmate.com/help/reference/ct_search_api_v1"+ ],+ 'apiKeyInstructions': [+ "Visit https://sslmate.com/signup?for=ct_search_api",+ "Register a new account with an email",+ "Navigate to https://sslmate.com/account/",+ "The API key is listed under 'API Credentials'",+ ],+ }+ }++ # Default options+ opts = {+ 'api_key': '',+ 'verify': True,+ 'max_pages': 20,+ 'certexpiringdays': 30+ }++ # Option descriptions+ optdescs = {+ 'api_key': 'CertSpotter API key.',+ 'verify': "Verify certificate subject alternative names resolve.",+ 'max_pages': "Maximum number of pages of results to fetch.",+ 'certexpiringdays': 'Number of days in the future a certificate expires to consider it as expiring.'+ }++ results = None+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = self.tempStorage()+ self.errorState = False++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['DOMAIN_NAME']++ # What events this module produces+ def producedEvents(self):+ return [+ 'INTERNET_NAME',+ 'INTERNET_NAME_UNRESOLVED',+ 'DOMAIN_NAME',+ 'AFFILIATE_INTERNET_NAME',+ 'AFFILIATE_INTERNET_NAME_UNRESOLVED',+ 'AFFILIATE_DOMAIN_NAME',+ 'SSL_CERTIFICATE_ISSUED',+ 'SSL_CERTIFICATE_ISSUER',+ 'SSL_CERTIFICATE_MISMATCH',+ 'SSL_CERTIFICATE_EXPIRED',+ 'SSL_CERTIFICATE_EXPIRING',+ 'SSL_CERTIFICATE_RAW',+ 'RAW_RIR_DATA'+ ]++ # Query CertSpotter issuances API endpoint+ def queryIssuances(self, domain, after=None):+ params = {+ 'domain': domain.encode('raw_unicode_escape').decode("ascii", errors='replace'),+ 'include_subdomains': 'true',+ 'match_wildcards': 'true',+ 'after': (after or '')+ }++ expand = '&expand='.join(['dns_names', 'issuer', 'cert'])++ headers = {+ 'Accept': 'application/json',+ 'Authorization': "Basic " + base64.b64encode(f"{self.opts['api_key']}:".encode('utf-8')).decode('utf-8')+ }++ res = self.sf.fetchUrl(+ f"https://api.certspotter.com/v1/issuances?{urllib.parse.urlencode(params)}&expand={expand}",+ headers=headers,+ timeout=15,+ useragent=self.opts['_useragent'],+ )++ # Free plan - 1,000 single-hostname queries / hour; 100 full-domain queries / hour+ time.sleep(1)++ if res['content'] is None:+ self.sf.debug('No response from CertSpotter API')+ return++ if res['code'] == '429':+ self.sf.error("You are being rate-limited by CertSpotter")+ self.errorState = True+ return++ if res['code'] != '200':+ self.sf.error(f"Unexpected HTTP response code {res['code']} from CertSpotter")+ self.errorState = True+ return++ try:+ data = json.loads(res['content'])+ except Exception as e:+ self.sf.debug(f"Error processing JSON response: {e}")+ return++ return data++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if eventData in self.results:+ return++ if self.errorState:+ return++ if self.opts['api_key'] == "":+ self.sf.error(f"You enabled {self.__class__.__name__} but did not set an API key!")+ self.errorState = True+ return++ self.results[eventData] = True++ self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")++ max_pages = int(self.opts['max_pages'])+ page = 1+ last_id = None+ hosts = list()+ while page <= max_pages:+ if self.checkForStop():+ break++ if self.errorState:+ break++ data = self.queryIssuances(eventData, last_id)++ if data is None or len(data) == 0:+ break++ page += 1++ evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)+ self.notifyListeners(evt)++ for result in data:+ cert_hosts = result.get('dns_names')++ if cert_hosts:+ for d in cert_hosts:+ if d != eventData:+ hosts.append(d.replace("*.", ""))++ if result.get('cert') is None:+ self.sf.debug('Response data contains no certificate data')+ continue++ try:+ dercert = result.get('cert').get('data')+ pemcert = self.sf.sslDerToPem(dercert)+ cert = self.sf.parseCert(pemcert, eventData, self.opts['certexpiringdays'])+ except Exception as e:+ self.sf.info(f"Error parsing certificate: {e}")+ continue++ if not cert.get('text'):+ self.sf.info("Failed to parse the SSL certificate")+ continue++ evt = SpiderFootEvent('SSL_CERTIFICATE_RAW', cert['text'], self.__name__, event)+ self.notifyListeners(evt)++ if cert.get('issuer'):+ evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert['issuer'], self.__name__, event)+ self.notifyListeners(evt)++ if cert.get('issued'):+ evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert['issued'], self.__name__, event)+ self.notifyListeners(evt)++ for san in set(cert.get('altnames', list())):+ hosts.append(san.replace("*.", ""))++ if cert.get('expired'):+ evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRED", cert.get('expirystr', 'Unknown'), self.__name__, event)+ self.notifyListeners(evt)+ continue++ if cert.get('expiring'):+ evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRING", cert.get('expirystr', 'Unknown'), self.__name__, event)+ self.notifyListeners(evt)+ continue++ # "To retrieve additional issuances, take the id field of the last issuance and pass it to the issuances endpoint in the after parameter"+ last_id = data[-1].get('id')++ if last_id is None:+ break++ if not hosts:+ return++ if self.opts['verify']:+ self.sf.info(f"Resolving {len(set(hosts))} hostnames ...")++ for domain in set(hosts):+ if self.checkForStop():+ return++ if domain in self.results:+ continue++ if self.getTarget().matches(domain, includeChildren=True, includeParents=True):+ evt_type = 'INTERNET_NAME'+ else:+ evt_type = 'AFFILIATE_INTERNET_NAME'++ if self.opts['verify'] and not self.sf.resolveHost(domain):+ self.sf.debug(f"Host {domain} could not be resolved")+ evt_type += '_UNRESOLVED'++ evt = SpiderFootEvent(evt_type, domain, self.__name__, event)+ self.notifyListeners(evt)++ if self.sf.isDomain(domain, self.opts['_internettlds']):+ if evt_type.startswith('AFFILIATE'):+ evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', domain, self.__name__, event)+ self.notifyListeners(evt)+ else:+ evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)+ self.notifyListeners(evt)++# End of sfp_certspotter classdiff --git a/test/unit/modules/test_sfp_certspotter.py b/test/unit/modules/test_sfp_certspotter.pynew file mode 100644index 00000000..ef1a4661--- /dev/null+++ b/test/unit/modules/test_sfp_certspotter.py@@ -0,0 +1,60 @@+# test_sfp_certspotter.py+import pytest+import unittest++from modules.sfp_certspotter import sfp_certspotter+from sflib import SpiderFoot+from spiderfoot import SpiderFootEvent, SpiderFootTarget+++class TestModulecertspotter(unittest.TestCase):+ """+ Test modules.sfp_certspotter+ """++ def test_opts(self):+ module = sfp_certspotter()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ """+ Test setup(self, sfc, userOpts=dict())+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_certspotter()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_certspotter()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_certspotter()+ self.assertIsInstance(module.producedEvents(), list)++ def test_handleEvent_no_api_key_should_set_errorState(self):+ """+ Test handleEvent(self, event)+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_certspotter()+ module.setup(sf, dict())++ target_value = 'example target value'+ target_type = 'INTERNET_NAME'+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = 'ROOT'+ event_data = 'example data'+ event_module = ''+ source_event = ''+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)+ self.assertTrue(module.errorState)