Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Add sfp_crobat_api module

Brendan Colese05e5db2f1c1
diff --git a/modules/sfp_crobat_api.py b/modules/sfp_crobat_api.py
new file mode 100644
index 00000000..9163751b
--- /dev/null
+++ b/modules/sfp_crobat_api.py
@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_crobat_api
+# Purpose: Search Crobat API for subdomains.
+# https://www.onsecurity.co.uk/blog/how-i-made-rapid7s-project-sonar-searchable
+# https://github.com/cgboal/sonarsearch
+#
+# Authors: <[email protected]>
+#
+# Created: 2020-08-29
+# Copyright: (c) bcoles 2020
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import json
+import time
+import urllib
+from sflib import SpiderFootPlugin, SpiderFootEvent
+
+class sfp_crobat_api(SpiderFootPlugin):
+
+ meta = {
+ 'name': "Crobat API",
+ 'summary': "Search Crobat API for subdomains.",
+ 'flags': [""],
+ 'useCases': ["Footprint", "Investigate", "Passive"],
+ 'categories': ["Passive DNS"]
+ }
+
+ opts = {
+ "verify": True,
+ "max_pages": 10,
+ "delay": 1
+ }
+
+ optdescs = {
+ "verify": "DNS resolve each identified subdomain.",
+ "max_pages": "Maximum number of pages of results to fetch.",
+ "delay": "Delay between requests, in seconds."
+ }
+
+ results = None
+ errorState = False
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.results = self.tempStorage()
+ self.errorState = False
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ def watchedEvents(self):
+ return ["DOMAIN_NAME"]
+
+ def producedEvents(self):
+ return ["RAW_RIR_DATA", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"]
+
+ def queryDomain(self, qry, page=0):
+ headers = {
+ "Accept": "application/json"
+ }
+ params = urllib.parse.urlencode({
+ 'page': page
+ })
+ domain = qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
+ res = self.sf.fetchUrl(
+ f"https://sonar.omnisint.io/subdomains/{domain}?{params}",
+ headers=headers,
+ timeout=30,
+ useragent=self.opts['_useragent']
+ )
+
+ time.sleep(self.opts['delay'])
+
+ return self.parseAPIResponse(res)
+
+ def parseAPIResponse(self, res):
+ # Future proofing - Crobat API does not implement rate limiting
+ if res['code'] == '429':
+ self.sf.error("You are being rate-limited by Crobat API", False)
+ self.errorState = True
+ return None
+
+ # Catch all non-200 status codes, and presume something went wrong
+ if res['code'] != '200':
+ self.sf.error("Failed to retrieve content from Crobat API", False)
+ self.errorState = True
+ return None
+
+ if res['content'] is None:
+ return None
+
+ # returns "null" when page has no data
+ if res['content'] == "null":
+ return None
+
+ try:
+ data = json.loads(res['content'])
+ except Exception as e:
+ self.sf.debug(f"Error processing JSON response: {e}")
+ return None
+
+ if not isinstance(data, list):
+ self.sf.error("Failed to retrieve content from Crobat API", False)
+ return None
+
+ return data
+
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ if self.errorState:
+ return None
+
+ if eventData in self.results:
+ return None
+
+ self.results[eventData] = True
+
+ self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
+
+ if eventName != "DOMAIN_NAME":
+ return None
+
+ page = 0
+ while page < self.opts['max_pages']:
+ if self.checkForStop():
+ return None
+
+ if self.errorState:
+ return None
+
+ data = self.queryDomain(eventData, page)
+
+ if not data:
+ self.sf.debug(f"No information found for domain {eventData} (page: {page})")
+ return None
+
+ evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
+ self.notifyListeners(evt)
+
+ page += 1
+
+ for domain in set(data):
+ if domain in self.results:
+ continue
+
+ if not self.getTarget().matches(domain, includeChildren=True, includeParents=True):
+ continue
+
+ if self.opts['verify'] and not self.sf.resolveHost(domain):
+ self.sf.debug(f"Host {domain} could not be resolved")
+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event)
+ self.notifyListeners(evt)
+ else:
+ evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event)
+ self.notifyListeners(evt)
+
+# End of sfp_crobat_api class
diff --git a/test/unit/modules/test_sfp_crobat_api.py b/test/unit/modules/test_sfp_crobat_api.py
new file mode 100644
index 00000000..f0a9722e
--- /dev/null
+++ b/test/unit/modules/test_sfp_crobat_api.py
@@ -0,0 +1,82 @@
+# test_sfp_crobat_api.py
+from sflib import SpiderFoot, SpiderFootEvent, SpiderFootTarget
+import unittest
+from modules.sfp_crobat_api import sfp_crobat_api
+
+class TestModuleCrobat_api(unittest.TestCase):
+ """
+ Test modules.sfp_crobat_api
+ """
+
+ default_options = {
+ '_debug': False, # Debug
+ '__logging': True, # Logging in general
+ '__outputfilter': None, # Event types to filter from modules' output
+ '__blocknotif': False, # Block notifications
+ '_fatalerrors': False,
+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
+ '_dnsserver': '', # Override the default resolver
+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch
+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
+ '_internettlds_cache': 72,
+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
+ '__version__': '3.2-DEV',
+ '__database': 'spiderfoot.test.db', # note: test database file
+ '__webaddr': '127.0.0.1',
+ '__webport': 5001,
+ '__docroot': '', # don't put trailing /
+ '__modules__': None, # List of modules. Will be set after start-up.
+ '_socks1type': '',
+ '_socks2addr': '',
+ '_socks3port': '',
+ '_socks4user': '',
+ '_socks5pwd': '',
+ '_socks6dns': True,
+ '_torctlport': 9051,
+ '__logstdout': False
+ }
+
+ def test_opts(self):
+ module = sfp_crobat_api()
+ self.assertEqual(len(module.opts), len(module.optdescs))
+
+ def test_setup(self):
+ """
+ Test setup(self, sfc, userOpts=dict())
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_crobat_api()
+ module.setup(sf, dict())
+
+ def test_watchedEvents_should_return_list(self):
+ module = sfp_crobat_api()
+ self.assertIsInstance(module.watchedEvents(), list)
+
+ def test_producedEvents_should_return_list(self):
+ module = sfp_crobat_api()
+ self.assertIsInstance(module.producedEvents(), list)
+
+ def test_handleEvent(self):
+ """
+ Test handleEvent(self, event)
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_crobat_api()
+ module.setup(sf, dict())
+
+ target_value = 'example target value'
+ target_type = 'IP_ADDRESS'
+ target = SpiderFootTarget(target_value, target_type)
+ module.setTarget(target)
+
+ event_type = 'ROOT'
+ event_data = 'example data'
+ event_module = ''
+ source_event = ''
+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
+
+ result = module.handleEvent(evt)
+
+ self.assertIsNone(result)