SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Added reversewhois.io module (#1169)
TheTechromancerfbbcbfe3e832
diff --git a/modules/sfp_reversewhois.py b/modules/sfp_reversewhois.pynew file mode 100644index 00000000..1199d2de--- /dev/null+++ b/modules/sfp_reversewhois.py@@ -0,0 +1,137 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_reversewhois+# Purpose: Scrape reversewhois.io+#+# Author: TheTechromancer+#+# Created: 05/20/2021+# Copyright: (c) Steve Micallef 2021+# Licence: GPL+# -------------------------------------------------------------------------------++import re+from bs4 import BeautifulSoup++from spiderfoot import SpiderFootEvent, SpiderFootPlugin+++class sfp_reversewhois(SpiderFootPlugin):++ meta = {+ "name": "ReverseWhois",+ "summary": "Reverse Whois lookups using reversewhois.io.",+ "useCases": ["Investigate", "Passive"],+ "categories": ["Search Engines"],+ "dataSource": {+ "website": "https://www.reversewhois.io/",+ "model": "FREE_NOAUTH_UNLIMITED",+ "favIcon": "https://www.reversewhois.io/dist/img/favicon-32x32.png",+ "description": "ReverseWhois is a free search engine to find domain names owned by an individual or company.\n"+ "Search based on names or email addresses.",+ }+ }++ # Default options+ opts = {}++ # Option descriptions+ optdescs = {}++ # Be sure to completely clear any class variables in setup()+ # or you run the risk of data persisting between scan runs.++ results = None+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = self.tempStorage()++ # Clear / reset any other class member variables here+ # or you risk them persisting between threads.++ for opt in list(userOpts.keys()):+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["DOMAIN_NAME"]++ # What events this module produces+ def producedEvents(self):+ return ["AFFILIATE_INTERNET_NAME", "AFFILIATE_DOMAIN_NAME", "DOMAIN_REGISTRAR"]++ # Search ReverseWhois+ def query(self, qry):+ url = f"https://reversewhois.io?searchterm={qry}"++ ret = ([], [])++ res = self.sf.fetchUrl(url, timeout=self.opts.get("_fetchtimeout", 30))++ if res["code"] not in ["200"]:+ self.sf.error("You may have exceeded ReverseWhois usage limits.")+ self.errorState = True+ return ret++ html = BeautifulSoup(res["content"])+ date_regex = re.compile(r'\d{4}-\d{2}-\d{2}')+ registrars = set()+ domains = set()+ for table_row in html.findAll("tr"):+ table_cells = table_row.findAll("td")+ # make double-sure we're in the right table by checking the date field+ try:+ if date_regex.match(table_cells[2].text.strip()):+ domain = table_cells[1].text.strip().lower()+ registrar = table_cells[-1].text.strip()+ if domain:+ domains.add(domain)+ if registrar:+ registrars.add(registrar)+ except IndexError:+ self.sf.debug(f"Invalid row {table_row}")+ continue++ ret = (list(domains), list(registrars))++ if not registrars and not domains:+ self.sf.info(f"No ReverseWhois info found for {qry}")++ return ret++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return++ self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug(f"Skipping {eventData}, already checked.")+ return+ else:+ self.results[eventData] = True++ domains, registrars = self.query(eventData)++ for domain in domains:+ # if this domain isn't the main target+ if not self.getTarget().matches(domain, includeChildren=False):+ e = SpiderFootEvent("AFFILIATE_INTERNET_NAME", domain, self.__name__, event)+ self.notifyListeners(e)+ if self.sf.isDomain(domain, self.opts["_internettlds"]):+ evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", domain, self.__name__, event)+ self.notifyListeners(evt)++ for registrar in registrars:+ e = SpiderFootEvent("DOMAIN_REGISTRAR", registrar, self.__name__, event)+ self.notifyListeners(e)++# End of sfp_reversewhois classdiff --git a/test/unit/modules/test_sfp_reversewhois.py b/test/unit/modules/test_sfp_reversewhois.pynew file mode 100644index 00000000..983ed4d1--- /dev/null+++ b/test/unit/modules/test_sfp_reversewhois.py@@ -0,0 +1,79 @@+# test_sfp_reversewhois.py+import unittest++from modules.sfp_reversewhois import sfp_reversewhois+from sflib import SpiderFoot+from spiderfoot import SpiderFootEvent, SpiderFootTarget+++class TestModulereversewhois(unittest.TestCase):+ """+ Test modules.sfp_reversewhois+ """++ default_options = {+ '_debug': False, # Debug+ '__logging': True, # Logging in general+ '__outputfilter': None, # Event types to filter from modules' output+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests+ '_dnsserver': '', # Override the default resolver+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',+ '_internettlds_cache': 72,+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",+ '__version__': '3.3',+ '__database': 'spiderfoot.test.db', # note: test database file+ '__modules__': None, # List of modules. Will be set after start-up.+ '_socks1type': '',+ '_socks2addr': '',+ '_socks3port': '',+ '_socks4user': '',+ '_socks5pwd': '',+ '_torctlport': 9051,+ '__logstdout': False+ }++ def test_opts(self):+ module = sfp_reversewhois()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ """+ Test setup(self, sfc, userOpts=dict())+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_reversewhois()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_reversewhois()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_reversewhois()+ self.assertIsInstance(module.producedEvents(), list)++ def test_handleEvent(self):+ """+ Test handleEvent(self, event)+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_reversewhois()+ module.setup(sf, dict())++ target_value = 'example target value'+ target_type = 'IP_ADDRESS'+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = 'ROOT'+ event_data = 'example data'+ event_module = ''+ source_event = ''+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)