SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name: sfp_c99
# Purpose: SpiderFoot plug-in that queries c99 API
#
# Author: Filip Aleksić <[email protected]>
#
# Created: 2020-08-27
# Copyright: (c) Steve Micallef
# Licence: MIT
# -------------------------------------------------------------------------------
import json
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
class sfp_c99(SpiderFootPlugin):
meta = {
"name": "C99",
"summary": "Queries the C99 API which offers various data (geo location, proxy detection, phone lookup, etc).",
'flags': ["apikey"],
"useCases": ["Footprint", "Passive", "Investigate"],
"categories": ["Search Engines"],
"dataSource": {
"website": "https://api.c99.nl/",
"model": "COMMERCIAL_ONLY",
"references": ["https://api.c99.nl/api_overview", "https://api.c99.nl/faq"],
"apiKeyInstructions": [
"Visit https://api.c99.nl",
"Click shop in the top navigation or go to https://api.c99.nl/dashboard/shop",
"Click purchase key on option 'C99.NL API KEY' (you can also purchase a 1 year key)",
"You will receive your API key by email.",
],
"favIcon": "https://api.c99.nl/favicon.ico",
"logo": "https://api.c99.nl/assets/images/logo.png",
"description": "C99 API service is versatile source of information. "
"They offer over 57 different APIs of which 10 are integrated in this module. "
"APIs that are integrated are subdomain finder, phone lookup, Skype resolver, "
"IP to Skype, firewall technology WAF detector, domain history, "
"IP to domains, IP geo location, proxy detector.",
},
}
opts = {
"api_key": "",
"verify": True,
"cohostsamedomain": False,
"maxcohost": 100,
}
optdescs = {
"api_key": "C99 API Key.",
"verify": "Verify identified domains still resolve to the associated specified IP address.",
"maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",
"cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?",
}
results = None
errorState = False
cohostcount = 0
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
self.cohostcount = 0
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
def watchedEvents(self):
return [
"DOMAIN_NAME",
"PHONE_NUMBER",
"IP_ADDRESS",
"USERNAME",
"EMAILADDR",
]
def producedEvents(self):
return [
"RAW_RIR_DATA",
"GEOINFO",
"INTERNET_NAME",
"INTERNET_NAME_UNRESOLVED",
"PROVIDER_TELCO",
"PHYSICAL_ADDRESS",
"PHYSICAL_COORDINATES",
"PROVIDER_DNS",
"IP_ADDRESS",
"USERNAME",
"ACCOUNT_EXTERNAL_OWNED",
"WEBSERVER_TECHNOLOGY",
"PROVIDER_HOSTING",
"CO_HOSTED_SITE"
]
def query(self, path, queryParam, queryData):
res = self.sf.fetchUrl(
f"https://api.c99.nl/{path}?key={self.opts['api_key']}&{queryParam}={queryData}&json",
timeout=self.opts["_fetchtimeout"],
useragent="SpiderFoot",
)
if res["code"] == "429":
self.error("Reaching rate limit on C99 API")
self.errorState = True
return None
if res["code"] == 400:
self.error("Invalid request or API key on C99 API")
self.errorState = True
return None
if res["content"] is None:
self.info(f"No C99 info found for {queryData}")
return None
try:
info = json.loads(res["content"])
except Exception as e:
self.errorState = True
self.error(f"Error processing response from C99: {e}")
return None
if not info.get('success', False):
return None
return info
def emitRawRirData(self, data, event):
evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
self.notifyListeners(evt)
def emitPhoneData(self, phoneData, event):
provider = phoneData.get("provider")
carrier = phoneData.get("carrier")
city = phoneData.get("city")
countryName = phoneData.get("country_name")
region = phoneData.get("region")
found = False
if provider or carrier:
evt = SpiderFootEvent(
"PROVIDER_TELCO",
f"Provider: {provider}, Carrier: {carrier}",
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if city or countryName or region:
evt = SpiderFootEvent(
"PHYSICAL_ADDRESS",
f"Country: {countryName}, Region: {region}, City: {city}",
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if found:
self.emitRawRirData(phoneData, event)
def emitSubDomainData(self, subDomainData, event):
found = False
for subDomainElem in subDomainData:
if self.checkForStop():
return
subDomain = subDomainElem.get("subdomain", "").strip()
if subDomain:
self.emitHostname(subDomain, event)
found = True
if found:
self.emitRawRirData(subDomainData, event)
def emitDomainHistoryData(self, domainHistoryData, event):
found = False
for domainHistoryElem in domainHistoryData:
if self.checkForStop():
return
ip = domainHistoryElem.get("ip_address")
if self.sf.validIP(ip):
evt = SpiderFootEvent(
"IP_ADDRESS",
ip,
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if found:
self.emitRawRirData(domainHistoryData, event)
def emitIpToSkypeData(self, data, event):
skype = data.get("skype")
if skype:
evt = SpiderFootEvent(
"ACCOUNT_EXTERNAL_OWNED",
f"Skype [{skype}]",
self.__name__,
event,
)
self.notifyListeners(evt)
evt = SpiderFootEvent(
"USERNAME",
skype,
self.__name__,
event,
)
self.notifyListeners(evt)
self.emitRawRirData(data, event)
def emitIpToDomainsData(self, data, event):
domains = data.get("domains")
found = False
if isinstance(domains, list):
for domain in domains:
if self.checkForStop():
return
domain = domain.strip()
if domain:
self.emitHostname(domain, event)
found = True
if found:
self.emitRawRirData(data, event)
def emitProxyDetectionData(self, data, event):
isProxy = data.get("proxy")
if isProxy:
evt = SpiderFootEvent(
"WEBSERVER_TECHNOLOGY",
f"Server is proxy: {isProxy}",
self.__name__,
event,
)
self.notifyListeners(evt)
self.emitRawRirData(data, event)
def emitGeoIPData(self, data, event):
found = False
hostName = data.get("hostname", "").strip()
if hostName:
self.emitHostname(hostName, event)
found = True
record = data.get("records")
if record:
country = record.get("country_name")
region = record["region"].get("name") if record.get("region") else None
city = record.get("city")
postalCode = record.get("postal_code")
latitude = record.get("latitude")
longitude = record.get("longitude")
provider = record.get("isp")
if provider:
evt = SpiderFootEvent(
"PROVIDER_HOSTING",
provider,
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if latitude and longitude:
evt = SpiderFootEvent(
"PHYSICAL_COORDINATES",
f"{latitude}, {longitude}",
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if region or country or city or postalCode:
evt = SpiderFootEvent(
"GEOINFO",
f"Country: {country}, Region: {region}, City: {city}, Postal code: {postalCode}",
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if found:
self.emitRawRirData(data, event)
def emitSkypeResolverData(self, data, event):
ip = data.get("ip")
ips = data.get("ips")
found = False
if ip and ip not in ips:
evt = SpiderFootEvent(
"IP_ADDRESS",
ip,
self.__name__,
event,
)
self.notifyListeners(evt)
found = True
if isinstance(ips, list):
found = True
for ipElem in ips:
if self.checkForStop():
return
evt = SpiderFootEvent(
"IP_ADDRESS",
ipElem.strip(),
self.__name__,
event,
)
self.notifyListeners(evt)
if found:
self.emitRawRirData(data, event)
def emitWafDetectorData(self, data, event):
firewall = data.get("result")
if firewall:
evt = SpiderFootEvent(
"WEBSERVER_TECHNOLOGY",
f"Firewall detected: {firewall}",
self.__name__,
event,
)
self.notifyListeners(evt)
self.emitRawRirData(data, event)
def emitHostname(self, data, event):
if not self.sf.validHost(data, self.opts['_internettlds']):
return
if self.opts["verify"] and not self.sf.resolveHost(data) and not self.sf.resolveHost6(data):
self.debug(f"Host {data} could not be resolved.")
if self.getTarget().matches(data):
evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", data, self.__name__, event)
self.notifyListeners(evt)
return
if self.getTarget().matches(data):
evt = SpiderFootEvent('INTERNET_NAME', data, self.__name__, event)
self.notifyListeners(evt)
if self.sf.isDomain(data, self.opts['_internettlds']):
evt = SpiderFootEvent('DOMAIN_NAME', data, self.__name__, event)
self.notifyListeners(evt)
return
if self.cohostcount < self.opts['maxcohost']:
if self.opts["verify"] and not self.sf.validateIP(data, event.data):
self.debug("Host no longer resolves to our IP.")
return
if not self.opts["cohostsamedomain"]:
if self.getTarget().matches(data, includeParents=True):
self.debug(
f"Skipping {data} because it is on the same domain."
)
return
if self.cohostcount < self.opts["maxcohost"]:
evt = SpiderFootEvent("CO_HOSTED_SITE", data, self.__name__, event)
self.notifyListeners(evt)
self.cohostcount += 1
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
# Once we are in this state, return immediately.
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if self.opts["api_key"] == "":
self.error("You enabled sfp_c99, but did not set an API key!")
self.errorState = True
return
# Don't look up stuff twice
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
self.results[eventData] = True
if eventName == "PHONE_NUMBER":
phoneData = self.query("phonelookup", "number", eventData)
phoneData = phoneData.get("details") if phoneData else None
if phoneData:
self.emitPhoneData(phoneData, event)
if eventName == "DOMAIN_NAME":
subDomainData = self.query("subdomainfinder", "domain", eventData)
subDomainData = (
subDomainData.get("subdomains") if subDomainData is not None else None
)
if isinstance(subDomainData, list):
self.emitSubDomainData(subDomainData, event)
domainHistoryData = self.query("domainhistory", "domain", eventData)
domainHistoryData = (
domainHistoryData.get("result") if domainHistoryData else None
)
if isinstance(domainHistoryData, list):
self.emitDomainHistoryData(domainHistoryData, event)
wafDetectorData = self.query("firewalldetector", "url", eventData)
if wafDetectorData:
self.emitWafDetectorData(wafDetectorData, event)
if eventName == "IP_ADDRESS":
ipToSkypeData = self.query("ip2skype", "ip", eventData)
if ipToSkypeData:
self.emitIpToSkypeData(ipToSkypeData, event)
ipToDomainsData = self.query("ip2domains", "ip", eventData)
if ipToDomainsData:
self.emitIpToDomainsData(ipToDomainsData, event)
proxyDetectionData = self.query("proxydetector", "ip", eventData)
if proxyDetectionData:
self.emitProxyDetectionData(proxyDetectionData, event)
geoIPData = self.query("geoip", "host", eventData)
if geoIPData:
self.emitGeoIPData(geoIPData, event)
if eventName == "USERNAME":
skypeResolverData = self.query("skyperesolver", "username", eventData)
if skypeResolverData:
self.emitSkypeResolverData(skypeResolverData, event)
# End of sfp_c99 class