Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Added Sublist3r PassiveDNS module (#1170)

TheTechromancer73489cf75c7c
diff --git a/modules/sfp_sublist3r.py b/modules/sfp_sublist3r.py
new file mode 100644
index 00000000..9eb2f10a
--- /dev/null
+++ b/modules/sfp_sublist3r.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_sublist3r
+# Purpose: SpiderFoot plug-in for subdomain enumeration using
+# api.sublist3r.com
+#
+# Author: TheTechromancer
+#
+# Created: 05/21/2021
+# Copyright: (c) Steve Micallef 2021
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import json
+
+from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+
+
+class sfp_sublist3r(SpiderFootPlugin):
+
+ meta = {
+ "name": "Sublist3r PassiveDNS",
+ "summary": "Passive subdomain enumeration using Sublist3r's API",
+ "useCases": ["Investigate", "Footprint", "Passive"],
+ "categories": ["Passive DNS"],
+ "dataSource": {
+ "website": "https://api.sublist3r.com",
+ "model": "FREE_NOAUTH_UNLIMITED",
+ "description": "This is the API queried by the Sublist3r tool.",
+ }
+ }
+
+ # Default options
+ opts = {}
+
+ # Option descriptions
+ optdescs = {}
+
+ results = None
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.sf.debug("Setting up sfp_sublist3r")
+ self.results = self.tempStorage()
+ self.opts.update(userOpts)
+
+ def watchedEvents(self):
+ return ["DOMAIN_NAME", "INTERNET_NAME"]
+
+ def producedEvents(self):
+ return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"]
+
+ def query(self, domain):
+ url = f"https://api.sublist3r.com/search.php?domain={domain}"
+ ret = []
+ res = self.sf.fetchUrl(
+ url,
+ useragent=self.opts.get("_useragent", "Spiderfoot"),
+ # mirror sublist3r's headers
+ headers={
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.8",
+ "Accept-Encoding": "gzip",
+ }
+ )
+ try:
+ ret = [s.strip().lower() for s in json.loads(res["content"])]
+ except json.decoder.JSONDecodeError as e:
+ self.sf.error(f"Error decoding JSON response: {e}")
+ except TypeError:
+ self.sf.error("Error querying Sublist3r API")
+
+ if res["code"] not in ["200"]:
+ self.sf.error(f"Bad response code \"{res['code']}\" from Sublist3r API")
+
+ return list(set(ret))
+
+ def sendEvent(self, source, host):
+ if self.sf.resolveHost(host):
+ e = SpiderFootEvent("INTERNET_NAME", host, self.__name__, source)
+ else:
+ e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, source)
+ self.notifyListeners(e)
+
+ def handleEvent(self, event):
+ query = str(event.data).lower()
+
+ self.sf.debug(f"Received event, {event.eventType}, from {event.module}")
+
+ # skip if we've already processed this event (or its parent domain/subdomain)
+ target = self.getTarget()
+ eventDataHash = self.sf.hashstring(query)
+ if eventDataHash in self.results or \
+ (target.matches(query, includeParents=True) and not
+ target.matches(query, includeChildren=False)):
+ self.sf.debug(f"Skipping already-processed event, {event.eventType}, from {event.module}")
+ return
+ self.results[eventDataHash] = True
+
+ for hostname in self.query(query):
+ if target.matches(hostname, includeParents=True) and not \
+ target.matches(hostname, includeChildren=False):
+ self.sendEvent(event, hostname)
+ else:
+ self.sf.debug(f"Invalid subdomain: {hostname}")
diff --git a/test/unit/modules/test_sfp_sublist3r.py b/test/unit/modules/test_sfp_sublist3r.py
new file mode 100644
index 00000000..7b7ccb3f
--- /dev/null
+++ b/test/unit/modules/test_sfp_sublist3r.py
@@ -0,0 +1,79 @@
+# test_sfp_sublist3r.py
+import unittest
+
+from modules.sfp_sublist3r import sfp_sublist3r
+from sflib import SpiderFoot
+from spiderfoot import SpiderFootEvent, SpiderFootTarget
+
+
+class TestModulesublist3r(unittest.TestCase):
+ """
+ Test modules.sfp_sublist3r
+ """
+
+ default_options = {
+ '_debug': False, # Debug
+ '__logging': True, # Logging in general
+ '__outputfilter': None, # Event types to filter from modules' output
+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
+ '_dnsserver': '', # Override the default resolver
+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch
+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
+ '_internettlds_cache': 72,
+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
+ '__version__': '3.3',
+ '__database': 'spiderfoot.test.db', # note: test database file
+ '__modules__': None, # List of modules. Will be set after start-up.
+ '_socks1type': '',
+ '_socks2addr': '',
+ '_socks3port': '',
+ '_socks4user': '',
+ '_socks5pwd': '',
+ '_torctlport': 9051,
+ '__logstdout': False
+ }
+
+ def test_opts(self):
+ module = sfp_sublist3r()
+ self.assertEqual(len(module.opts), len(module.optdescs))
+
+ def test_setup(self):
+ """
+ Test setup(self, sfc, userOpts=dict())
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_sublist3r()
+ module.setup(sf, dict())
+
+ def test_watchedEvents_should_return_list(self):
+ module = sfp_sublist3r()
+ self.assertIsInstance(module.watchedEvents(), list)
+
+ def test_producedEvents_should_return_list(self):
+ module = sfp_sublist3r()
+ self.assertIsInstance(module.producedEvents(), list)
+
+ def test_handleEvent(self):
+ """
+ Test handleEvent(self, event)
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_sublist3r()
+ module.setup(sf, dict())
+
+ target_value = 'example target value'
+ target_type = 'IP_ADDRESS'
+ target = SpiderFootTarget(target_value, target_type)
+ module.setTarget(target)
+
+ event_type = 'ROOT'
+ event_data = 'example data'
+ event_module = ''
+ source_event = ''
+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
+
+ result = module.handleEvent(evt)
+
+ self.assertIsNone(result)