Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_censys: Add support for IPv6; report software and operating system (#1691)

bcolesb0df7efa9b66
diff --git a/modules/sfp_censys.py b/modules/sfp_censys.py
index ed913bab..cefea543 100644
--- a/modules/sfp_censys.py
+++ b/modules/sfp_censys.py
@@ -35,7 +35,7 @@ class sfp_censys(SpiderFootPlugin):
'website': "https://censys.io/",
'model': "FREE_AUTH_LIMITED",
'references': [
- "https://censys.io/api",
+ "https://search.censys.io/api",
"https://search.censys.io/search/language",
"https://github.com/censys/censys-postman/blob/main/Censys_Search.postman_collection.json",
],
@@ -61,6 +61,7 @@ class sfp_censys(SpiderFootPlugin):
'delay': 3,
'netblocklookup': True,
'maxnetblock': 24,
+ 'maxv6netblock': 120,
"age_limit_days": 90,
}
@@ -70,6 +71,7 @@ class sfp_censys(SpiderFootPlugin):
'delay': 'Delay between requests, in seconds.',
'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?",
'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)",
+ 'maxv6netblock': "If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)",
"age_limit_days": "Ignore any records older than this many days. 0 = unlimited.",
}
@@ -84,15 +86,24 @@ class sfp_censys(SpiderFootPlugin):
self.opts[opt] = userOpts[opt]
def watchedEvents(self):
- return ["IP_ADDRESS", "NETBLOCK_OWNER"]
+ return [
+ "IP_ADDRESS",
+ "IPV6_ADDRESS",
+ "NETBLOCK_OWNER",
+ "NETBLOCKV6_OWNER",
+ ]
def producedEvents(self):
return [
"BGP_AS_MEMBER",
+ "UDP_PORT_OPEN",
"TCP_PORT_OPEN",
+ "TCP_PORT_OPEN_BANNER",
"OPERATING_SYSTEM",
+ "SOFTWARE_USED",
"WEBSERVER_HTTPHEADERS",
"NETBLOCK_MEMBER",
+ "NETBLOCKV6_MEMBER",
"GEOINFO",
"RAW_RIR_DATA"
]
@@ -188,37 +199,41 @@ class sfp_censys(SpiderFootPlugin):
return data
def handleEvent(self, event):
- eventName = event.eventType
- srcModuleName = event.module
- eventData = event.data
-
if self.errorState:
return
- self.debug(f"Received event, {eventName}, from {srcModuleName}")
+ eventName = event.eventType
+
+ self.debug(f"Received event, {eventName}, from {event.module}")
if self.opts['censys_api_key_uid'] == "" or self.opts['censys_api_key_secret'] == "":
self.error(f"You enabled {self.__class__.__name__} but did not set an API uid/secret!")
self.errorState = True
return
+ eventData = event.data
+
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
- if eventName == 'NETBLOCK_OWNER':
+ if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:
if not self.opts['netblocklookup']:
return
- max_netblock = self.opts['maxnetblock']
+ if eventName == 'NETBLOCKV6_OWNER':
+ max_netblock = self.opts['maxv6netblock']
+ else:
+ max_netblock = self.opts['maxnetblock']
+
if IPNetwork(eventData).prefixlen < max_netblock:
self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}")
return
qrylist = list()
- if eventName.startswith("NETBLOCK_"):
+ if eventName.startswith("NETBLOCK"):
for ipaddr in IPNetwork(eventData):
qrylist.append(str(ipaddr))
self.results[str(ipaddr)] = True
@@ -242,11 +257,13 @@ class sfp_censys(SpiderFootPlugin):
self.debug(f"Found results for {addr} in Censys.io")
- # For netblocks, we need to create the IP address event so that
- # the threat intel event is more meaningful.
+ # For netblocks, we need to create the associated IP address event first.
if eventName == 'NETBLOCK_OWNER':
pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event)
self.notifyListeners(pevent)
+ elif eventName == 'NETBLOCKV6_OWNER':
+ pevent = SpiderFootEvent("IPV6_ADDRESS", addr, self.__name__, event)
+ self.notifyListeners(pevent)
else:
pevent = event
@@ -263,7 +280,7 @@ class sfp_censys(SpiderFootPlugin):
self.debug(f"Record found but too old ({created_dt}), skipping.")
continue
except Exception as e:
- self.error(f"Error encountered processing last_updated_at record for {eventData} ({e})")
+ self.error(f"Error encountered processing last_updated_at record for {addr}: {e}")
try:
location = rec.get('location')
@@ -275,6 +292,7 @@ class sfp_censys(SpiderFootPlugin):
location.get('province'),
location.get('postal_code'),
location.get('country'),
+ location.get('continent'),
] if _f
]
)
@@ -282,30 +300,69 @@ class sfp_censys(SpiderFootPlugin):
e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, pevent)
self.notifyListeners(e)
except Exception as e:
- self.error(f"Error encountered processing location record for {eventData} ({e})")
+ self.error(f"Error encountered processing location record for {addr}: {e}")
try:
services = rec.get('services')
if services:
+ softwares = list()
+ tcp_banners = list()
for service in services:
port = service.get('port')
if port:
- evt = SpiderFootEvent("TCP_PORT_OPEN", f"{addr}:{port}", self.__name__, pevent)
- self.notifyListeners(evt)
-
- try:
- headers = service['http']['response']['headers']
- except Exception:
- headers = None
-
- if headers:
- dat = json.dumps(headers, ensure_ascii=False)
- e = SpiderFootEvent("WEBSERVER_HTTPHEADERS", dat, self.__name__, pevent)
- e.actualSource = addr
- self.notifyListeners(e)
+ transport_protocol = service.get('transport_protocol')
+ banner = service.get('banner')
+
+ if transport_protocol == "UDP":
+ evt = SpiderFootEvent("UDP_PORT_OPEN", f"{addr}:{port}", self.__name__, pevent)
+ self.notifyListeners(evt)
+ elif transport_protocol == "TCP":
+ evt = SpiderFootEvent("TCP_PORT_OPEN", f"{addr}:{port}", self.__name__, pevent)
+ self.notifyListeners(evt)
+ if banner:
+ tcp_banners.append(banner)
+
+ software = service.get('software', list())
+ if software:
+ for sw in software:
+ s = ' '.join(
+ filter(
+ None,
+ [
+ sw.get('vendor'),
+ sw.get('product'),
+ sw.get('version')
+ ]
+ )
+ )
+ if s:
+ softwares.append(s)
+
+ http = service.get('http')
+ if http:
+ response = http.get('response')
+ if response:
+ headers = response.get('headers')
+ if headers:
+ e = SpiderFootEvent(
+ "WEBSERVER_HTTPHEADERS",
+ json.dumps(headers, ensure_ascii=False),
+ self.__name__,
+ pevent
+ )
+ e.actualSource = addr
+ self.notifyListeners(e)
+
+ for software in set(softwares):
+ evt = SpiderFootEvent("SOFTWARE_USED", software, self.__name__, pevent)
+ self.notifyListeners(evt)
+
+ for banner in set(tcp_banners):
+ evt = SpiderFootEvent("TCP_PORT_OPEN_BANNER", str(banner), self.__name__, pevent)
+ self.notifyListeners(evt)
except Exception as e:
- self.error(f"Error encountered processing services record for {eventData} ({e})")
+ self.error(f"Error encountered processing services record for {addr}: {e}")
try:
autonomous_system = rec.get('autonomous_system')
@@ -316,23 +373,34 @@ class sfp_censys(SpiderFootPlugin):
self.notifyListeners(e)
bgp_prefix = autonomous_system.get('bgp_prefix')
- if bgp_prefix:
- e = SpiderFootEvent("NETBLOCK_MEMBER", str(bgp_prefix), self.__name__, pevent)
+ if bgp_prefix and self.sf.validIpNetwork(bgp_prefix):
+ if ':' in bgp_prefix:
+ e = SpiderFootEvent("NETBLOCKV6_MEMBER", str(bgp_prefix), self.__name__, pevent)
+ else:
+ e = SpiderFootEvent("NETBLOCK_MEMBER", str(bgp_prefix), self.__name__, pevent)
self.notifyListeners(e)
except Exception as e:
- self.error(f"Error encountered processing autonomous_system record for {eventData} ({e})")
+ self.error(f"Error encountered processing autonomous_system record for {addr}: {e}")
try:
operating_system = rec.get('operating_system')
if operating_system:
- vendor = operating_system.get('vendor')
- product = operating_system.get('product')
- os = ' '.join(filter(None, [vendor, product]))
+ os = ' '.join(
+ filter(
+ None,
+ [
+ operating_system.get('vendor'),
+ operating_system.get('product'),
+ operating_system.get('version'),
+ operating_system.get('edition')
+ ]
+ )
+ )
if os:
e = SpiderFootEvent("OPERATING_SYSTEM", os, self.__name__, pevent)
self.notifyListeners(e)
except Exception as e:
- self.error(f"Error encountered processing operating_system record for {eventData} ({e})")
+ self.error(f"Error encountered processing operating_system record for {addr}: {e}")
# End of sfp_censys class