SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Add sfp_crobat_api module
Brendan Colese05e5db2f1c1
diff --git a/modules/sfp_crobat_api.py b/modules/sfp_crobat_api.pynew file mode 100644index 00000000..9163751b--- /dev/null+++ b/modules/sfp_crobat_api.py@@ -0,0 +1,162 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_crobat_api+# Purpose: Search Crobat API for subdomains.+# https://www.onsecurity.co.uk/blog/how-i-made-rapid7s-project-sonar-searchable+# https://github.com/cgboal/sonarsearch+#+# Authors: <[email protected]>+#+# Created: 2020-08-29+# Copyright: (c) bcoles 2020+# Licence: GPL+# -------------------------------------------------------------------------------++import json+import time+import urllib+from sflib import SpiderFootPlugin, SpiderFootEvent++class sfp_crobat_api(SpiderFootPlugin):++ meta = {+ 'name': "Crobat API",+ 'summary': "Search Crobat API for subdomains.",+ 'flags': [""],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"]+ }++ opts = {+ "verify": True,+ "max_pages": 10,+ "delay": 1+ }++ optdescs = {+ "verify": "DNS resolve each identified subdomain.",+ "max_pages": "Maximum number of pages of results to fetch.",+ "delay": "Delay between requests, in seconds."+ }++ results = None+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = self.tempStorage()+ self.errorState = False++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ def watchedEvents(self):+ return ["DOMAIN_NAME"]++ def producedEvents(self):+ return ["RAW_RIR_DATA", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"]++ def queryDomain(self, qry, page=0):+ headers = {+ "Accept": "application/json"+ }+ params = urllib.parse.urlencode({+ 'page': page+ })+ domain = qry.encode('raw_unicode_escape').decode("ascii", errors='replace')+ res = self.sf.fetchUrl(+ f"https://sonar.omnisint.io/subdomains/{domain}?{params}",+ headers=headers,+ timeout=30,+ useragent=self.opts['_useragent']+ )++ time.sleep(self.opts['delay'])++ return self.parseAPIResponse(res)++ def parseAPIResponse(self, res):+ # Future proofing - Crobat API does not implement rate limiting+ if res['code'] == '429':+ self.sf.error("You are being rate-limited by Crobat API", False)+ self.errorState = True+ return None++ # Catch all non-200 status codes, and presume something went wrong+ if res['code'] != '200':+ self.sf.error("Failed to retrieve content from Crobat API", False)+ self.errorState = True+ return None++ if res['content'] is None:+ return None++ # returns "null" when page has no data+ if res['content'] == "null":+ return None++ try:+ data = json.loads(res['content'])+ except Exception as e:+ self.sf.debug(f"Error processing JSON response: {e}")+ return None++ if not isinstance(data, list):+ self.sf.error("Failed to retrieve content from Crobat API", False)+ return None++ return data++ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return None++ if eventData in self.results:+ return None++ self.results[eventData] = True++ self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")++ if eventName != "DOMAIN_NAME":+ return None++ page = 0+ while page < self.opts['max_pages']:+ if self.checkForStop():+ return None++ if self.errorState:+ return None++ data = self.queryDomain(eventData, page)++ if not data:+ self.sf.debug(f"No information found for domain {eventData} (page: {page})")+ return None++ evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)+ self.notifyListeners(evt)++ page += 1++ for domain in set(data):+ if domain in self.results:+ continue++ if not self.getTarget().matches(domain, includeChildren=True, includeParents=True):+ continue++ if self.opts['verify'] and not self.sf.resolveHost(domain):+ self.sf.debug(f"Host {domain} could not be resolved")+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event)+ self.notifyListeners(evt)+ else:+ evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event)+ self.notifyListeners(evt)++# End of sfp_crobat_api classdiff --git a/test/unit/modules/test_sfp_crobat_api.py b/test/unit/modules/test_sfp_crobat_api.pynew file mode 100644index 00000000..f0a9722e--- /dev/null+++ b/test/unit/modules/test_sfp_crobat_api.py@@ -0,0 +1,82 @@+# test_sfp_crobat_api.py+from sflib import SpiderFoot, SpiderFootEvent, SpiderFootTarget+import unittest+from modules.sfp_crobat_api import sfp_crobat_api++class TestModuleCrobat_api(unittest.TestCase):+ """+ Test modules.sfp_crobat_api+ """++ default_options = {+ '_debug': False, # Debug+ '__logging': True, # Logging in general+ '__outputfilter': None, # Event types to filter from modules' output+ '__blocknotif': False, # Block notifications+ '_fatalerrors': False,+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests+ '_dnsserver': '', # Override the default resolver+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',+ '_internettlds_cache': 72,+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",+ '__version__': '3.2-DEV',+ '__database': 'spiderfoot.test.db', # note: test database file+ '__webaddr': '127.0.0.1',+ '__webport': 5001,+ '__docroot': '', # don't put trailing /+ '__modules__': None, # List of modules. Will be set after start-up.+ '_socks1type': '',+ '_socks2addr': '',+ '_socks3port': '',+ '_socks4user': '',+ '_socks5pwd': '',+ '_socks6dns': True,+ '_torctlport': 9051,+ '__logstdout': False+ }++ def test_opts(self):+ module = sfp_crobat_api()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ """+ Test setup(self, sfc, userOpts=dict())+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_crobat_api()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_crobat_api()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_crobat_api()+ self.assertIsInstance(module.producedEvents(), list)++ def test_handleEvent(self):+ """+ Test handleEvent(self, event)+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_crobat_api()+ module.setup(sf, dict())++ target_value = 'example target value'+ target_type = 'IP_ADDRESS'+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = 'ROOT'+ event_data = 'example data'+ event_module = ''+ source_event = ''+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)