SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Feature/ipregistry.co (#1004)
Leo Trubach7b58168cef74
diff --git a/modules/sfp_ipregistry.py b/modules/sfp_ipregistry.pynew file mode 100644index 00000000..b05ff7e4--- /dev/null+++ b/modules/sfp_ipregistry.py@@ -0,0 +1,168 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_ipregistry+# Purpose: ipregistry database query module.+#+# Author: Leo Trubach <[email protected]>+#+# Created: 2020-09-08+# Copyright: (c) Steve Micallef+# Licence: GPL+# -------------------------------------------------------------------------------++import json+import urllib.parse++from spiderfoot import SpiderFootEvent, SpiderFootPlugin+++class sfp_ipregistry(SpiderFootPlugin):+ meta = {+ "name": "ipregistry",+ "summary": "Query ipregistry database",+ "flags": ["apikey"],+ "useCases": ["Passive"],+ "categories": ["Reputation Systems"],+ "dataSource": {+ "website": "https://ipregistry.co/",+ "model": "FREE_AUTH_LIMITED",+ "references": ["https://ipregistry.co/docs"],+ "apiKeyInstructions": [+ "Visit https://dashboard.ipregistry.co/signup",+ "Register a free account",+ "Click on 'API Keys' in left navbar",+ "Click on 'Click to reveal API key' for existing Default key",+ ],+ "favIcon": "https://cdn.ipregistry.co/icons/favicon-32x32.png",+ "logo": "https://ipregistry.co/assets/ipregistry.svg",+ "description": "Ipregistry is a trusted and in-depth IP "+ "Geolocation and Threat detections source of information that can"+ "benefit publishers, ad networks, retailers, financial services, "+ "e-commerce stores and more.",+ },+ }++ opts = {+ "api_key": "",+ }++ optdescs = {+ "api_key": "Ipregistry API Key.",+ }++ results = None++ errorState = False++ def setup(self, sfc, userOpts=None):+ self.sf = sfc+ self.results = self.tempStorage()++ if userOpts:+ self.opts.update(userOpts)++ def watchedEvents(self):+ return ["IP_ADDRESS", "IPV6_ADDRESS"]++ def producedEvents(self):+ return ["GEOINFO", "MALICIOUS_IPADDR", "PHYSICAL_COORDINATES", "RAW_RIR_DATA"]++ def query(self, qry):+ qs = urllib.parse.urlencode({"key": self.opts["api_key"]})+ res = self.sf.fetchUrl(+ f"https://api.ipregistry.co/{qry}?{qs}",+ timeout=self.opts["_fetchtimeout"],+ useragent="SpiderFoot",+ )++ if res["content"] is None:+ self.sf.info(f"No {self.meta['name']} info found for {qry}")+ return None++ try:+ info = json.loads(res["content"])+ except Exception as e:+ self.sf.error(+ f"Error processing JSON response from {self.meta['name']}: {e}", False+ )+ return None++ return info++ def emit(self, etype, data, pevent, notify=True):+ evt = SpiderFootEvent(etype, data, self.__name__, pevent)+ if notify:+ self.notifyListeners(evt)+ return evt++ def generate_location_events(self, location, pevent):+ if not isinstance(location, dict):+ return+ physical_location = None+ country = location.get("country")+ if isinstance(country, dict):+ country_name = country.get("name")+ else:+ country_name = None+ region = location.get("region")+ if isinstance(region, dict):+ region_name = region.get("name")+ else:+ region_name = None+ latitude = location.get("latitude")+ longitude = location.get("longitude")+ if latitude and longitude:+ physical_location = f"{latitude}, {longitude}"+ geo_info = ", ".join(+ [+ _f+ for _f in [+ location.get("city"),+ region_name,+ location.get("postal"),+ country_name,+ ]+ if _f+ ]+ )+ if geo_info:+ self.emit("GEOINFO", geo_info, pevent)+ if physical_location:+ self.emit("PHYSICAL_COORDINATES", physical_location, pevent)++ def generate_security_events(self, security, pevent):+ if not isinstance(security, dict):+ return+ malicious = any(+ security.get(k) for k in ("is_abuser", "is_attacker", "is_threat")+ )+ if malicious:+ self.emit("MALICIOUS_IPADDR", f"ipregistry [{pevent.data}]", pevent)++ def generate_events(self, data, pevent):+ if not isinstance(data, dict):+ return+ self.generate_location_events(data.get("location"), pevent)+ self.generate_security_events(data.get("security"), pevent)++ def handleEvent(self, event):+ if self.errorState:+ return None++ self.sf.debug(f"Received event, {event.eventType}, from {event.module}")++ if self.opts["api_key"] == "":+ self.sf.error(f"You enabled {self.__class__.__name__} but did not set an API key!", False)+ self.errorState = True+ return None++ if event.data in self.results:+ self.sf.debug(f"Skipping {event.data}, already checked.")+ return None+ self.results[event.data] = True++ if event.eventType in ("IP_ADDRESS", "IPV6_ADDRESS"):+ data = self.query(event.data)+ self.generate_events(data, event)++ self.emit("RAW_RIR_DATA", json.dumps(data), event)diff --git a/modules/sfp_template.py b/modules/sfp_template.pyindex 75574706..cf3efc86 100644--- a/modules/sfp_template.py+++ b/modules/sfp_template.py@@ -50,7 +50,7 @@ class sfp_template(SpiderFootPlugin):# - Passive DNS: Analyses passive DNS sources# - Public Registries: Queries open/public registries of information# - Real World: Queries sources about the real world (addresses, names, etc.)- # - Reputation Systems: Queries systems that describe the reptuation of other systems+ # - Reputation Systems: Queries systems that describe the reputation of other systems# - Search Engines: Searches public search engines with data about the whole Internet# - Secondary Networks: Queries information about participation on secondary networks, like Bitcoin# - Social Media: Searches social media data sources@@ -192,7 +192,7 @@ class sfp_template(SpiderFootPlugin):# Clear / reset any other class member variables here# or you risk them persisting between threads.- # The data souce for a module is, by defualt, set to the module name.+ # The data source for a module is, by default, set to the module name.# If you want to override that, for instance in cases where the module# is purely processing data from other modules instead of producing# data itself, you can do so with the following. Note that this is onlydiff --git a/test/unit/modules/test_sfp_ipregistry.py b/test/unit/modules/test_sfp_ipregistry.pynew file mode 100644index 00000000..87fcfa8c--- /dev/null+++ b/test/unit/modules/test_sfp_ipregistry.py@@ -0,0 +1,83 @@+# test_sfp_ipregistry.py+import unittest++from modules.sfp_ipregistry import sfp_ipregistry+from sflib import SpiderFoot+from spiderfoot import SpiderFootEvent, SpiderFootTarget+++class TestModuletemplate(unittest.TestCase):+ """+ Test modules.sfp_ipregistry+ """++ default_options = {+ "_debug": False, # Debug+ "__logging": True, # Logging in general+ "__outputfilter": None, # Event types to filter from modules' output+ "_useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0", # User-Agent to use for HTTP requests+ "_dnsserver": "", # Override the default resolver+ "_fetchtimeout": 5, # number of seconds before giving up on a fetch+ "_internettlds": "https://publicsuffix.org/list/effective_tld_names.dat",+ "_internettlds_cache": 72,+ "_genericusers": "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",+ "__version__": "3.3-DEV",+ "__database": "spiderfoot.test.db", # note: test database file+ "__modules__": None, # List of modules. Will be set after start-up.+ "_socks1type": "",+ "_socks2addr": "",+ "_socks3port": "",+ "_socks4user": "",+ "_socks5pwd": "",+ "_torctlport": 9051,+ "__logstdout": False,+ }++ def test_opts(self):+ module = sfp_ipregistry()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ """+ Test setup(self, sfc, userOpts=dict())+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_ipregistry()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_ipregistry()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_ipregistry()+ self.assertIsInstance(module.producedEvents(), list)++ def test_handleEvent(self):+ """+ Test handleEvent(self, event)+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_ipregistry()+ module.setup(sf, dict())++ target_value = "example target value"+ target_type = "IP_ADDRESS"+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = "ROOT"+ event_data = "example data"+ event_module = ""+ source_event = ""+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)+++if __name__ == "__main__":+ unittest.main()