Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Feature/ipregistry.co (#1004)

Leo Trubach7b58168cef74
diff --git a/modules/sfp_ipregistry.py b/modules/sfp_ipregistry.py
new file mode 100644
index 00000000..b05ff7e4
--- /dev/null
+++ b/modules/sfp_ipregistry.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_ipregistry
+# Purpose: ipregistry database query module.
+#
+# Author: Leo Trubach <[email protected]>
+#
+# Created: 2020-09-08
+# Copyright: (c) Steve Micallef
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import json
+import urllib.parse
+
+from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+
+
+class sfp_ipregistry(SpiderFootPlugin):
+ meta = {
+ "name": "ipregistry",
+ "summary": "Query ipregistry database",
+ "flags": ["apikey"],
+ "useCases": ["Passive"],
+ "categories": ["Reputation Systems"],
+ "dataSource": {
+ "website": "https://ipregistry.co/",
+ "model": "FREE_AUTH_LIMITED",
+ "references": ["https://ipregistry.co/docs"],
+ "apiKeyInstructions": [
+ "Visit https://dashboard.ipregistry.co/signup",
+ "Register a free account",
+ "Click on 'API Keys' in left navbar",
+ "Click on 'Click to reveal API key' for existing Default key",
+ ],
+ "favIcon": "https://cdn.ipregistry.co/icons/favicon-32x32.png",
+ "logo": "https://ipregistry.co/assets/ipregistry.svg",
+ "description": "Ipregistry is a trusted and in-depth IP "
+ "Geolocation and Threat detections source of information that can"
+ "benefit publishers, ad networks, retailers, financial services, "
+ "e-commerce stores and more.",
+ },
+ }
+
+ opts = {
+ "api_key": "",
+ }
+
+ optdescs = {
+ "api_key": "Ipregistry API Key.",
+ }
+
+ results = None
+
+ errorState = False
+
+ def setup(self, sfc, userOpts=None):
+ self.sf = sfc
+ self.results = self.tempStorage()
+
+ if userOpts:
+ self.opts.update(userOpts)
+
+ def watchedEvents(self):
+ return ["IP_ADDRESS", "IPV6_ADDRESS"]
+
+ def producedEvents(self):
+ return ["GEOINFO", "MALICIOUS_IPADDR", "PHYSICAL_COORDINATES", "RAW_RIR_DATA"]
+
+ def query(self, qry):
+ qs = urllib.parse.urlencode({"key": self.opts["api_key"]})
+ res = self.sf.fetchUrl(
+ f"https://api.ipregistry.co/{qry}?{qs}",
+ timeout=self.opts["_fetchtimeout"],
+ useragent="SpiderFoot",
+ )
+
+ if res["content"] is None:
+ self.sf.info(f"No {self.meta['name']} info found for {qry}")
+ return None
+
+ try:
+ info = json.loads(res["content"])
+ except Exception as e:
+ self.sf.error(
+ f"Error processing JSON response from {self.meta['name']}: {e}", False
+ )
+ return None
+
+ return info
+
+ def emit(self, etype, data, pevent, notify=True):
+ evt = SpiderFootEvent(etype, data, self.__name__, pevent)
+ if notify:
+ self.notifyListeners(evt)
+ return evt
+
+ def generate_location_events(self, location, pevent):
+ if not isinstance(location, dict):
+ return
+ physical_location = None
+ country = location.get("country")
+ if isinstance(country, dict):
+ country_name = country.get("name")
+ else:
+ country_name = None
+ region = location.get("region")
+ if isinstance(region, dict):
+ region_name = region.get("name")
+ else:
+ region_name = None
+ latitude = location.get("latitude")
+ longitude = location.get("longitude")
+ if latitude and longitude:
+ physical_location = f"{latitude}, {longitude}"
+ geo_info = ", ".join(
+ [
+ _f
+ for _f in [
+ location.get("city"),
+ region_name,
+ location.get("postal"),
+ country_name,
+ ]
+ if _f
+ ]
+ )
+ if geo_info:
+ self.emit("GEOINFO", geo_info, pevent)
+ if physical_location:
+ self.emit("PHYSICAL_COORDINATES", physical_location, pevent)
+
+ def generate_security_events(self, security, pevent):
+ if not isinstance(security, dict):
+ return
+ malicious = any(
+ security.get(k) for k in ("is_abuser", "is_attacker", "is_threat")
+ )
+ if malicious:
+ self.emit("MALICIOUS_IPADDR", f"ipregistry [{pevent.data}]", pevent)
+
+ def generate_events(self, data, pevent):
+ if not isinstance(data, dict):
+ return
+ self.generate_location_events(data.get("location"), pevent)
+ self.generate_security_events(data.get("security"), pevent)
+
+ def handleEvent(self, event):
+ if self.errorState:
+ return None
+
+ self.sf.debug(f"Received event, {event.eventType}, from {event.module}")
+
+ if self.opts["api_key"] == "":
+ self.sf.error(f"You enabled {self.__class__.__name__} but did not set an API key!", False)
+ self.errorState = True
+ return None
+
+ if event.data in self.results:
+ self.sf.debug(f"Skipping {event.data}, already checked.")
+ return None
+ self.results[event.data] = True
+
+ if event.eventType in ("IP_ADDRESS", "IPV6_ADDRESS"):
+ data = self.query(event.data)
+ self.generate_events(data, event)
+
+ self.emit("RAW_RIR_DATA", json.dumps(data), event)
diff --git a/modules/sfp_template.py b/modules/sfp_template.py
index 75574706..cf3efc86 100644
--- a/modules/sfp_template.py
+++ b/modules/sfp_template.py
@@ -50,7 +50,7 @@ class sfp_template(SpiderFootPlugin):
# - Passive DNS: Analyses passive DNS sources
# - Public Registries: Queries open/public registries of information
# - Real World: Queries sources about the real world (addresses, names, etc.)
- # - Reputation Systems: Queries systems that describe the reptuation of other systems
+ # - Reputation Systems: Queries systems that describe the reputation of other systems
# - Search Engines: Searches public search engines with data about the whole Internet
# - Secondary Networks: Queries information about participation on secondary networks, like Bitcoin
# - Social Media: Searches social media data sources
@@ -192,7 +192,7 @@ class sfp_template(SpiderFootPlugin):
# Clear / reset any other class member variables here
# or you risk them persisting between threads.
- # The data souce for a module is, by defualt, set to the module name.
+ # The data source for a module is, by default, set to the module name.
# If you want to override that, for instance in cases where the module
# is purely processing data from other modules instead of producing
# data itself, you can do so with the following. Note that this is only
diff --git a/test/unit/modules/test_sfp_ipregistry.py b/test/unit/modules/test_sfp_ipregistry.py
new file mode 100644
index 00000000..87fcfa8c
--- /dev/null
+++ b/test/unit/modules/test_sfp_ipregistry.py
@@ -0,0 +1,83 @@
+# test_sfp_ipregistry.py
+import unittest
+
+from modules.sfp_ipregistry import sfp_ipregistry
+from sflib import SpiderFoot
+from spiderfoot import SpiderFootEvent, SpiderFootTarget
+
+
+class TestModuletemplate(unittest.TestCase):
+ """
+ Test modules.sfp_ipregistry
+ """
+
+ default_options = {
+ "_debug": False, # Debug
+ "__logging": True, # Logging in general
+ "__outputfilter": None, # Event types to filter from modules' output
+ "_useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0", # User-Agent to use for HTTP requests
+ "_dnsserver": "", # Override the default resolver
+ "_fetchtimeout": 5, # number of seconds before giving up on a fetch
+ "_internettlds": "https://publicsuffix.org/list/effective_tld_names.dat",
+ "_internettlds_cache": 72,
+ "_genericusers": "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
+ "__version__": "3.3-DEV",
+ "__database": "spiderfoot.test.db", # note: test database file
+ "__modules__": None, # List of modules. Will be set after start-up.
+ "_socks1type": "",
+ "_socks2addr": "",
+ "_socks3port": "",
+ "_socks4user": "",
+ "_socks5pwd": "",
+ "_torctlport": 9051,
+ "__logstdout": False,
+ }
+
+ def test_opts(self):
+ module = sfp_ipregistry()
+ self.assertEqual(len(module.opts), len(module.optdescs))
+
+ def test_setup(self):
+ """
+ Test setup(self, sfc, userOpts=dict())
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_ipregistry()
+ module.setup(sf, dict())
+
+ def test_watchedEvents_should_return_list(self):
+ module = sfp_ipregistry()
+ self.assertIsInstance(module.watchedEvents(), list)
+
+ def test_producedEvents_should_return_list(self):
+ module = sfp_ipregistry()
+ self.assertIsInstance(module.producedEvents(), list)
+
+ def test_handleEvent(self):
+ """
+ Test handleEvent(self, event)
+ """
+ sf = SpiderFoot(self.default_options)
+
+ module = sfp_ipregistry()
+ module.setup(sf, dict())
+
+ target_value = "example target value"
+ target_type = "IP_ADDRESS"
+ target = SpiderFootTarget(target_value, target_type)
+ module.setTarget(target)
+
+ event_type = "ROOT"
+ event_data = "example data"
+ event_module = ""
+ source_event = ""
+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
+
+ result = module.handleEvent(evt)
+
+ self.assertIsNone(result)
+
+
+if __name__ == "__main__":
+ unittest.main()