SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Added Sublist3r PassiveDNS module (#1170)
TheTechromancer73489cf75c7c
diff --git a/modules/sfp_sublist3r.py b/modules/sfp_sublist3r.pynew file mode 100644index 00000000..9eb2f10a--- /dev/null+++ b/modules/sfp_sublist3r.py@@ -0,0 +1,105 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_sublist3r+# Purpose: SpiderFoot plug-in for subdomain enumeration using+# api.sublist3r.com+#+# Author: TheTechromancer+#+# Created: 05/21/2021+# Copyright: (c) Steve Micallef 2021+# Licence: GPL+# -------------------------------------------------------------------------------++import json++from spiderfoot import SpiderFootEvent, SpiderFootPlugin+++class sfp_sublist3r(SpiderFootPlugin):++ meta = {+ "name": "Sublist3r PassiveDNS",+ "summary": "Passive subdomain enumeration using Sublist3r's API",+ "useCases": ["Investigate", "Footprint", "Passive"],+ "categories": ["Passive DNS"],+ "dataSource": {+ "website": "https://api.sublist3r.com",+ "model": "FREE_NOAUTH_UNLIMITED",+ "description": "This is the API queried by the Sublist3r tool.",+ }+ }++ # Default options+ opts = {}++ # Option descriptions+ optdescs = {}++ results = None++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.sf.debug("Setting up sfp_sublist3r")+ self.results = self.tempStorage()+ self.opts.update(userOpts)++ def watchedEvents(self):+ return ["DOMAIN_NAME", "INTERNET_NAME"]++ def producedEvents(self):+ return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"]++ def query(self, domain):+ url = f"https://api.sublist3r.com/search.php?domain={domain}"+ ret = []+ res = self.sf.fetchUrl(+ url,+ useragent=self.opts.get("_useragent", "Spiderfoot"),+ # mirror sublist3r's headers+ headers={+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",+ "Accept-Language": "en-US,en;q=0.8",+ "Accept-Encoding": "gzip",+ }+ )+ try:+ ret = [s.strip().lower() for s in json.loads(res["content"])]+ except json.decoder.JSONDecodeError as e:+ self.sf.error(f"Error decoding JSON response: {e}")+ except TypeError:+ self.sf.error("Error querying Sublist3r API")++ if res["code"] not in ["200"]:+ self.sf.error(f"Bad response code \"{res['code']}\" from Sublist3r API")++ return list(set(ret))++ def sendEvent(self, source, host):+ if self.sf.resolveHost(host):+ e = SpiderFootEvent("INTERNET_NAME", host, self.__name__, source)+ else:+ e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, source)+ self.notifyListeners(e)++ def handleEvent(self, event):+ query = str(event.data).lower()++ self.sf.debug(f"Received event, {event.eventType}, from {event.module}")++ # skip if we've already processed this event (or its parent domain/subdomain)+ target = self.getTarget()+ eventDataHash = self.sf.hashstring(query)+ if eventDataHash in self.results or \+ (target.matches(query, includeParents=True) and not+ target.matches(query, includeChildren=False)):+ self.sf.debug(f"Skipping already-processed event, {event.eventType}, from {event.module}")+ return+ self.results[eventDataHash] = True++ for hostname in self.query(query):+ if target.matches(hostname, includeParents=True) and not \+ target.matches(hostname, includeChildren=False):+ self.sendEvent(event, hostname)+ else:+ self.sf.debug(f"Invalid subdomain: {hostname}")diff --git a/test/unit/modules/test_sfp_sublist3r.py b/test/unit/modules/test_sfp_sublist3r.pynew file mode 100644index 00000000..7b7ccb3f--- /dev/null+++ b/test/unit/modules/test_sfp_sublist3r.py@@ -0,0 +1,79 @@+# test_sfp_sublist3r.py+import unittest++from modules.sfp_sublist3r import sfp_sublist3r+from sflib import SpiderFoot+from spiderfoot import SpiderFootEvent, SpiderFootTarget+++class TestModulesublist3r(unittest.TestCase):+ """+ Test modules.sfp_sublist3r+ """++ default_options = {+ '_debug': False, # Debug+ '__logging': True, # Logging in general+ '__outputfilter': None, # Event types to filter from modules' output+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests+ '_dnsserver': '', # Override the default resolver+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',+ '_internettlds_cache': 72,+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",+ '__version__': '3.3',+ '__database': 'spiderfoot.test.db', # note: test database file+ '__modules__': None, # List of modules. Will be set after start-up.+ '_socks1type': '',+ '_socks2addr': '',+ '_socks3port': '',+ '_socks4user': '',+ '_socks5pwd': '',+ '_torctlport': 9051,+ '__logstdout': False+ }++ def test_opts(self):+ module = sfp_sublist3r()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ """+ Test setup(self, sfc, userOpts=dict())+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_sublist3r()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_sublist3r()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_sublist3r()+ self.assertIsInstance(module.producedEvents(), list)++ def test_handleEvent(self):+ """+ Test handleEvent(self, event)+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_sublist3r()+ module.setup(sf, dict())++ target_value = 'example target value'+ target_type = 'IP_ADDRESS'+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = 'ROOT'+ event_data = 'example data'+ event_module = ''+ source_event = ''+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)