Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Add sfp_certspotter (#1268)

bcolesffc47770cf4b
diff --git a/modules/sfp_certspotter.py b/modules/sfp_certspotter.py
new file mode 100644
index 00000000..fa5af1fe
--- /dev/null
+++ b/modules/sfp_certspotter.py
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_certspotter
+# Purpose: Gather information about SSL certificates from SSLMate CertSpotter API.
+#
+# Author: <[email protected]>
+#
+# Created: 2021-08-15
+# Copyright: (c) bcoles
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import base64
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+
+
+class sfp_certspotter(SpiderFootPlugin):
+ meta = {
+ 'name': "CertSpotter",
+ 'summary': "Gather information about SSL certificates from SSLMate CertSpotter API.",
+ 'flags': ["apikey"],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Crawling and Scanning"],
+ 'dataSource': {
+ 'website': "https://sslmate.com/certspotter/",
+ 'model': "FREE_AUTH_LIMITED",
+ 'references': [
+ "https://sslmate.com/help/reference/ct_search_api_v1"
+ ],
+ 'apiKeyInstructions': [
+ "Visit https://sslmate.com/signup?for=ct_search_api",
+ "Register a new account with an email",
+ "Navigate to https://sslmate.com/account/",
+ "The API key is listed under 'API Credentials'",
+ ],
+ }
+ }
+
+ # Default options
+ opts = {
+ 'api_key': '',
+ 'verify': True,
+ 'max_pages': 20,
+ 'certexpiringdays': 30
+ }
+
+ # Option descriptions
+ optdescs = {
+ 'api_key': 'CertSpotter API key.',
+ 'verify': "Verify certificate subject alternative names resolve.",
+ 'max_pages': "Maximum number of pages of results to fetch.",
+ 'certexpiringdays': 'Number of days in the future a certificate expires to consider it as expiring.'
+ }
+
+ results = None
+ errorState = False
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.results = self.tempStorage()
+ self.errorState = False
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ['DOMAIN_NAME']
+
+ # What events this module produces
+ def producedEvents(self):
+ return [
+ 'INTERNET_NAME',
+ 'INTERNET_NAME_UNRESOLVED',
+ 'DOMAIN_NAME',
+ 'AFFILIATE_INTERNET_NAME',
+ 'AFFILIATE_INTERNET_NAME_UNRESOLVED',
+ 'AFFILIATE_DOMAIN_NAME',
+ 'SSL_CERTIFICATE_ISSUED',
+ 'SSL_CERTIFICATE_ISSUER',
+ 'SSL_CERTIFICATE_MISMATCH',
+ 'SSL_CERTIFICATE_EXPIRED',
+ 'SSL_CERTIFICATE_EXPIRING',
+ 'SSL_CERTIFICATE_RAW',
+ 'RAW_RIR_DATA'
+ ]
+
+ # Query CertSpotter issuances API endpoint
+ def queryIssuances(self, domain, after=None):
+ params = {
+ 'domain': domain.encode('raw_unicode_escape').decode("ascii", errors='replace'),
+ 'include_subdomains': 'true',
+ 'match_wildcards': 'true',
+ 'after': (after or '')
+ }
+
+ expand = '&expand='.join(['dns_names', 'issuer', 'cert'])
+
+ headers = {
+ 'Accept': 'application/json',
+ 'Authorization': "Basic " + base64.b64encode(f"{self.opts['api_key']}:".encode('utf-8')).decode('utf-8')
+ }
+
+ res = self.sf.fetchUrl(
+ f"https://api.certspotter.com/v1/issuances?{urllib.parse.urlencode(params)}&expand={expand}",
+ headers=headers,
+ timeout=15,
+ useragent=self.opts['_useragent'],
+ )
+
+ # Free plan - 1,000 single-hostname queries / hour; 100 full-domain queries / hour
+ time.sleep(1)
+
+ if res['content'] is None:
+ self.sf.debug('No response from CertSpotter API')
+ return
+
+ if res['code'] == '429':
+ self.sf.error("You are being rate-limited by CertSpotter")
+ self.errorState = True
+ return
+
+ if res['code'] != '200':
+ self.sf.error(f"Unexpected HTTP response code {res['code']} from CertSpotter")
+ self.errorState = True
+ return
+
+ try:
+ data = json.loads(res['content'])
+ except Exception as e:
+ self.sf.debug(f"Error processing JSON response: {e}")
+ return
+
+ return data
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ if eventData in self.results:
+ return
+
+ if self.errorState:
+ return
+
+ if self.opts['api_key'] == "":
+ self.sf.error(f"You enabled {self.__class__.__name__} but did not set an API key!")
+ self.errorState = True
+ return
+
+ self.results[eventData] = True
+
+ self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
+
+ max_pages = int(self.opts['max_pages'])
+ page = 1
+ last_id = None
+ hosts = list()
+ while page <= max_pages:
+ if self.checkForStop():
+ break
+
+ if self.errorState:
+ break
+
+ data = self.queryIssuances(eventData, last_id)
+
+ if data is None or len(data) == 0:
+ break
+
+ page += 1
+
+ evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
+ self.notifyListeners(evt)
+
+ for result in data:
+ cert_hosts = result.get('dns_names')
+
+ if cert_hosts:
+ for d in cert_hosts:
+ if d != eventData:
+ hosts.append(d.replace("*.", ""))
+
+ if result.get('cert') is None:
+ self.sf.debug('Response data contains no certificate data')
+ continue
+
+ try:
+ dercert = result.get('cert').get('data')
+ pemcert = self.sf.sslDerToPem(dercert)
+ cert = self.sf.parseCert(pemcert, eventData, self.opts['certexpiringdays'])
+ except Exception as e:
+ self.sf.info(f"Error parsing certificate: {e}")
+ continue
+
+ if not cert.get('text'):
+ self.sf.info("Failed to parse the SSL certificate")
+ continue
+
+ evt = SpiderFootEvent('SSL_CERTIFICATE_RAW', cert['text'], self.__name__, event)
+ self.notifyListeners(evt)
+
+ if cert.get('issuer'):
+ evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert['issuer'], self.__name__, event)
+ self.notifyListeners(evt)
+
+ if cert.get('issued'):
+ evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert['issued'], self.__name__, event)
+ self.notifyListeners(evt)
+
+ for san in set(cert.get('altnames', list())):
+ hosts.append(san.replace("*.", ""))
+
+ if cert.get('expired'):
+ evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRED", cert.get('expirystr', 'Unknown'), self.__name__, event)
+ self.notifyListeners(evt)
+ continue
+
+ if cert.get('expiring'):
+ evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRING", cert.get('expirystr', 'Unknown'), self.__name__, event)
+ self.notifyListeners(evt)
+ continue
+
+ # "To retrieve additional issuances, take the id field of the last issuance and pass it to the issuances endpoint in the after parameter"
+ last_id = data[-1].get('id')
+
+ if last_id is None:
+ break
+
+ if not hosts:
+ return
+
+ if self.opts['verify']:
+ self.sf.info(f"Resolving {len(set(hosts))} hostnames ...")
+
+ for domain in set(hosts):
+ if self.checkForStop():
+ return
+
+ if domain in self.results:
+ continue
+
+ if self.getTarget().matches(domain, includeChildren=True, includeParents=True):
+ evt_type = 'INTERNET_NAME'
+ else:
+ evt_type = 'AFFILIATE_INTERNET_NAME'
+
+ if self.opts['verify'] and not self.sf.resolveHost(domain):
+ self.sf.debug(f"Host {domain} could not be resolved")
+ evt_type += '_UNRESOLVED'
+
+ evt = SpiderFootEvent(evt_type, domain, self.__name__, event)
+ self.notifyListeners(evt)
+
+ if self.sf.isDomain(domain, self.opts['_internettlds']):
+ if evt_type.startswith('AFFILIATE'):
+ evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', domain, self.__name__, event)
+ self.notifyListeners(evt)
+ else:
+ evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)
+ self.notifyListeners(evt)
+
+# End of sfp_certspotter class
diff --git a/test/unit/modules/test_sfp_certspotter.py b/test/unit/modules/test_sfp_certspotter.py
new file mode 100644
index 00000000..ef1a4661
--- /dev/null
+++ b/test/unit/modules/test_sfp_certspotter.py
@@ -0,0 +1,60 @@
+# test_sfp_certspotter.py
+import pytest
+import unittest
+
+from modules.sfp_certspotter import sfp_certspotter
+from sflib import SpiderFoot
+from spiderfoot import SpiderFootEvent, SpiderFootTarget
+
+
+class TestModulecertspotter(unittest.TestCase):
+ """
+ Test modules.sfp_certspotter
+ """
+
+ def test_opts(self):
+ module = sfp_certspotter()
+ self.assertEqual(len(module.opts), len(module.optdescs))
+
+ def test_setup(self):
+ """
+ Test setup(self, sfc, userOpts=dict())
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_certspotter()
+ module.setup(sf, dict())
+
+ def test_watchedEvents_should_return_list(self):
+ module = sfp_certspotter()
+ self.assertIsInstance(module.watchedEvents(), list)
+
+ def test_producedEvents_should_return_list(self):
+ module = sfp_certspotter()
+ self.assertIsInstance(module.producedEvents(), list)
+
+ def test_handleEvent_no_api_key_should_set_errorState(self):
+ """
+ Test handleEvent(self, event)
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_certspotter()
+ module.setup(sf, dict())
+
+ target_value = 'example target value'
+ target_type = 'INTERNET_NAME'
+ target = SpiderFootTarget(target_value, target_type)
+ module.setTarget(target)
+
+ event_type = 'ROOT'
+ event_data = 'example data'
+ event_module = ''
+ source_event = ''
+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
+
+ result = module.handleEvent(evt)
+
+ self.assertIsNone(result)
+ self.assertTrue(module.errorState)