Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_spyse: Cleanup and add support for IPv6 addresses (#1475)

bcoles4d5fb30472d6
diff --git a/modules/sfp_spyse.py b/modules/sfp_spyse.py
index 41dcea49..a59be290 100644
--- a/modules/sfp_spyse.py
+++ b/modules/sfp_spyse.py
@@ -21,7 +21,7 @@ from spiderfoot import SpiderFootEvent, SpiderFootPlugin
class sfp_spyse(SpiderFootPlugin):
meta = {
'name': "Spyse",
- 'summary': "Search Spyse.com Internet assets registry for information about domains, IPv4 hosts, potential vulnerabilities, passive DNS, etc.",
+ 'summary': "Search Spyse.com Internet assets registry for information about domains, IP addresses, host info, potential vulnerabilities, passive DNS, etc.",
'flags': ["apikey"],
'useCases': ["Footprint", "Investigate", "Passive"],
'categories': ["Search Engines"],
@@ -29,7 +29,8 @@ class sfp_spyse(SpiderFootPlugin):
'website': "https://spyse.com",
'model': "FREE_AUTH_LIMITED",
'references': [
- "https://spyse.com/api"
+ "https://spyse.com/api",
+ "https://spyse-dev.readme.io/reference/quick-start",
],
'apiKeyInstructions': [
"Visit https://spyse.com",
@@ -86,19 +87,43 @@ class sfp_spyse(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return ["IP_ADDRESS", "DOMAIN_NAME", "INTERNET_NAME"]
+ return [
+ "IP_ADDRESS",
+ "IPV6_ADDRESS",
+ "DOMAIN_NAME",
+ "INTERNET_NAME",
+ ]
# What events this module produces
def producedEvents(self):
- return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "DOMAIN_NAME",
- "IP_ADDRESS", "IPV6_ADDRESS", "CO_HOSTED_SITE",
- "RAW_RIR_DATA", "TCP_PORT_OPEN", "OPERATING_SYSTEM",
- "WEBSERVER_BANNER", "WEBSERVER_HTTPHEADERS",
- "COMPANY_NAME", "WEBSERVER_TECHNOLOGY", "PROVIDER_MAIL",
- "PROVIDER_DNS", "DNS_TEXT", "DNS_SPF", "COUNTRY_NAME",
- "WEB_ANALYTICS_ID", "VULNERABILITY", "SSL_CERTIFICATE_ISSUED",
- "SSL_CERTIFICATE_ISSUER", "EMAILADDR", "DOMAIN_REGISTRAR",
- "HTTP_CODE"]
+ return [
+ "INTERNET_NAME",
+ "INTERNET_NAME_UNRESOLVED",
+ "DOMAIN_NAME",
+ "IP_ADDRESS",
+ "IPV6_ADDRESS",
+ "CO_HOSTED_SITE",
+ "RAW_RIR_DATA",
+ "TCP_PORT_OPEN",
+ "TCP_PORT_OPEN_BANNER",
+ "OPERATING_SYSTEM",
+ "WEBSERVER_HTTPHEADERS",
+ "SOFTWARE_USED",
+ "COMPANY_NAME",
+ "WEBSERVER_TECHNOLOGY",
+ "PROVIDER_MAIL",
+ "PROVIDER_DNS",
+ "DNS_TEXT",
+ "DNS_SPF",
+ "GEOINFO",
+ "WEB_ANALYTICS_ID",
+ "VULNERABILITY",
+ "SSL_CERTIFICATE_ISSUED",
+ "SSL_CERTIFICATE_ISSUER",
+ "EMAILADDR",
+ "DOMAIN_REGISTRAR",
+ "HTTP_CODE",
+ ]
def querySubdomains(self, qry, currentOffset):
"""Query subdomains of domain
@@ -112,7 +137,6 @@ class sfp_spyse(SpiderFootPlugin):
Returns:
dict: JSON formatted results
"""
-
headers = {
'Accept': "application/json",
'Content-Type': "application/json",
@@ -154,7 +178,6 @@ class sfp_spyse(SpiderFootPlugin):
Returns:
dict: JSON formatted results
"""
-
headers = {
'Accept': "application/json",
'Authorization': "Bearer " + self.opts['api_key']
@@ -171,7 +194,7 @@ class sfp_spyse(SpiderFootPlugin):
return self.parseAPIResponse(res)
def queryIPPort(self, qry):
- """Query IP port lookup
+ """Query IPv4 port lookup
https://spyse-dev.readme.io/reference/ips#ip_details
@@ -181,6 +204,9 @@ class sfp_spyse(SpiderFootPlugin):
Returns:
dict: JSON formatted results
"""
+ if not self.sf.validIP(qry):
+ self.info(f"Invalid IPv4 address: {qry}")
+ return None
headers = {
'Accept': "application/json",
@@ -203,22 +229,30 @@ class sfp_spyse(SpiderFootPlugin):
https://spyse-dev.readme.io/reference/domains#domain_search
Args:
- qry (str): IP address
+ qry (str): IPv4/IPv6 address
currentOffset (int): start from this search result offset
Returns:
dict: JSON formatted results
"""
-
headers = {
'Accept': "application/json",
'Content-Type': "application/json",
'Authorization': "Bearer " + self.opts['api_key']
}
+
+ if self.sf.validIP(qry):
+ search_key = 'dns_a'
+ elif self.sf.validIP6(qry):
+ search_key = 'dns_aaaa'
+ else:
+ self.info(f"Invalid IP address: {qry}")
+ return None
+
body = {
"search_params": [
{
- "dns_a": {
+ search_key: {
"operator": "eq",
"value": f"{qry.encode('raw_unicode_escape').decode('ascii', errors='replace')}"
}
@@ -229,10 +263,10 @@ class sfp_spyse(SpiderFootPlugin):
}
res = self.sf.fetchUrl(
'https://api.spyse.com/v4/data/domain/search',
- postData=json.dumps(body),
headers=headers,
timeout=15,
- useragent=self.opts['_useragent']
+ useragent=self.opts['_useragent'],
+ postData=json.dumps(body)
)
time.sleep(self.opts['delay'])
@@ -245,24 +279,31 @@ class sfp_spyse(SpiderFootPlugin):
https://spyse-dev.readme.io/reference/quick-start
Args:
- res: TBD
+ res: Spyse API HTTP response
Returns:
dict: JSON formatted results
"""
-
if res['code'] == '400':
self.error("Malformed request")
return None
+ if res['code'] == '401':
+ self.error("Unauthorized")
+ self.errorState = True
+ return None
+
if res['code'] == '402':
self.error("Request limit exceeded")
self.errorState = True
return None
if res['code'] == '403':
- self.error("Authentication failed")
- self.errorState = True
+ if res['content'] and "the applied search parameter isn't allowed for your subscription plan" in res['content']:
+ self.error("The applied search parameter isn't allowed for your subscription plan")
+ else:
+ self.error("Authentication failed")
+ self.errorState = True
return None
# Future proofing - Spyse does not implement rate limiting
@@ -286,106 +327,79 @@ class sfp_spyse(SpiderFootPlugin):
self.debug(f"Error processing JSON response: {e}")
return None
- if data.get('message'):
- self.debug("Received error from Spyse: " + data.get('message'))
+ if data.get('error'):
+ self.info(f"Received error from Spyse: {data.get('error')}")
return data
- # Report extra data in the record
- def reportExtraData(self, record, event):
- # Note: 'operation_system' is the correct key (not 'operating_system')
- operatingSystem = record.get('operation_system')
- if operatingSystem:
- evt = SpiderFootEvent('OPERATING_SYSTEM', operatingSystem, self.__name__, event)
- self.notifyListeners(evt)
-
- webServer = record.get('product')
- if webServer:
- evt = SpiderFootEvent('WEBSERVER_BANNER', webServer, self.__name__, event)
- self.notifyListeners(evt)
-
- httpHeaders = record.get('http_headers')
- if httpHeaders:
- evt = SpiderFootEvent('WEBSERVER_HTTPHEADERS', httpHeaders, self.__name__, event)
- self.notifyListeners(evt)
-
- # Handle events sent to this module
def handleEvent(self, event):
-
if self.errorState:
return
if self.opts['api_key'] == '':
- self.error("You enabled sfp_spyse but did not set an API key!")
+ self.error(f"You enabled {self.__class__.__name__} but did not set an API key!")
self.errorState = True
return
- eventName = event.eventType
- srcModuleName = event.module
- eventData = event.data
- if eventData in self.results:
+ if event.data in self.results:
return
- self.results[eventData] = True
+ self.results[event.data] = True
- self.debug(f"Received event, {eventName}, from {srcModuleName}")
+ self.debug(f"Received event, {event.eventType}, from {event.module}")
- # Query open ports for source IP Address
- if eventName in ["IP_ADDRESS"]:
- if self.checkForStop():
- return
- self.process_ip_event(event)
+ if event.eventType == 'IP_ADDRESS':
+ self.retrieve_cohosts(event)
+ self.retrieve_ports(event)
+ elif event.eventType == "IPV6_ADDRESS":
+ self.retrieve_cohosts(event)
+ elif event.eventType in ["DOMAIN_NAME", "INTERNET_NAME"]:
+ self.retrieve_domain_info(event)
+ self.retrieve_subdomains(event)
+ else:
+ self.debug(f"Unexpected event type {event.eventType}, skipping")
- # Query subdomains
- if eventName in ["DOMAIN_NAME", "INTERNET_NAME"]:
- if self.checkForStop():
- return
- self.process_domain_event(event)
-
- def process_ip_event(self, event):
+ def retrieve_cohosts(self, event):
cohosts = list()
currentOffset = 0
- nextPageHasData = True
eventData = event.data
- while nextPageHasData:
+ while True:
if self.checkForStop():
- return
+ break
+
+ if self.errorState:
+ break
data = self.queryDomainsOnIP(eventData, currentOffset)
if not data:
- nextPageHasData = False
break
data = data.get("data")
if data is None:
- self.debug("No domains found on IP address " + eventData)
- nextPageHasData = False
+ self.debug(f"No co-hosts found on IP address {eventData}")
break
- else:
- records = data.get('items')
- if records:
- for record in records:
- domain = record.get('name')
- if domain:
- evt = SpiderFootEvent('RAW_RIR_DATA', str(record), self.__name__, event)
- self.notifyListeners(evt)
- cohosts.append(domain)
- self.reportExtraData(record, event)
+ records = data.get('items')
+ if records:
+ for record in records:
+ domain = record.get('name')
+ if domain:
+ evt = SpiderFootEvent('RAW_RIR_DATA', str(record), self.__name__, event)
+ self.notifyListeners(evt)
+ cohosts.append(domain)
# Calculate if there are any records in the next offset (page)
if len(records) < self.limit:
- nextPageHasData = False
+ break
currentOffset += self.limit
for co in set(cohosts):
-
if co in self.results:
continue
if self.opts['verify'] and not self.sf.validateIP(co, eventData):
- self.debug("Host " + co + " no longer resolves to " + eventData)
+ self.debug(f"Host {co} no longer resolves to {eventData}")
continue
if not self.opts['cohostsamedomain']:
@@ -402,228 +416,281 @@ class sfp_spyse(SpiderFootPlugin):
self.notifyListeners(evt)
self.cohostcount += 1
- ports = list()
+ def retrieve_ports(self, event):
+ eventData = event.data
data = self.queryIPPort(eventData)
- if data:
- data = data.get("data")
+ if not data:
+ return
- if data is None:
- self.debug("No open ports found for IP " + eventData)
- else:
- records = data.get('items')
- if records:
-
- for record in records:
- if record.get("ports"):
-
- for port_data in record["ports"]:
-
- port = port_data.get('port')
- if port:
- evt = SpiderFootEvent('RAW_RIR_DATA', str(record), self.__name__, event)
- self.notifyListeners(evt)
-
- ports.append(str(eventData) + ":" + str(port))
- self.reportExtraData(record, event)
- if record.get("geo_info"):
- country = record["geo_info"].get("country")
- if country:
- evt = SpiderFootEvent('COUNTRY_NAME', country, self.__name__, event)
- self.notifyListeners(evt)
- if record.get("cve_list"):
- for cve in record["cve_list"]:
- evt = SpiderFootEvent('VULNERABILITY', cve["id"], self.__name__, event)
- self.notifyListeners(evt)
- if record.get("technologies"):
- for tech in record["technologies"]:
- if tech.get("version"):
- evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY',
- f"{tech['name']} {tech['version']}",
- self.__name__, event)
- else:
- evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', tech['name'], self.__name__,
- event)
- self.notifyListeners(evt)
-
- for port in ports:
- if port in self.results:
+ data = data.get("data")
+
+ if data is None:
+ self.debug(f"No open ports found for IP {eventData}")
+ return
+
+ records = data.get('items')
+ if not records:
+ return
+
+ for record in records:
+ geoinfo = record.get("geo_info")
+ if geoinfo:
+ city = geoinfo.get("city")
+ country = geoinfo.get("country")
+ location = ', '.join([_f for _f in [city, country] if _f])
+ if location:
+ evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)
+ self.notifyListeners(evt)
+
+ cve_list = record.get("cve_list")
+ if cve_list:
+ for cve in cve_list:
+ cve_id = cve.get('id')
+ if cve_id:
+ evt = SpiderFootEvent('VULNERABILITY', cve_id, self.__name__, event)
+ self.notifyListeners(evt)
+
+ technologies = record.get("technologies")
+ if technologies:
+ for tech in technologies:
+ name = tech.get("name")
+
+ if not name:
+ continue
+
+ version = tech.get("version")
+ software = ' '.join(filter(None, [name, version]))
+ if software:
+ evt = SpiderFootEvent('SOFTWARE_USED', f"{software}", self.__name__, event)
+ self.notifyListeners(evt)
+
+ ports = record.get('ports')
+ if not ports:
+ continue
+
+ for port_data in ports:
+ if not port_data:
continue
- self.results[port] = True
- evt = SpiderFootEvent('TCP_PORT_OPEN', str(port), self.__name__, event)
+ evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(port_data), self.__name__, event)
self.notifyListeners(evt)
- def process_domain_event(self, event):
- currentOffset = 0
- nextPageHasData = True
- domains = list()
+ port = port_data.get('port')
+
+ if not port:
+ continue
+
+ ip_port = f"{eventData}:{port}"
+
+ if ip_port in self.results:
+ continue
+
+ self.results[ip_port] = True
+
+ port_event = SpiderFootEvent('TCP_PORT_OPEN', ip_port, self.__name__, event)
+ self.notifyListeners(port_event)
+
+ banner = port_data.get('banner')
+ if banner:
+ evt = SpiderFootEvent('TCP_PORT_OPEN_BANNER', json.dumps(banner), self.__name__, port_event)
+ self.notifyListeners(evt)
+
+ def retrieve_domain_info(self, event):
eventData = event.data
if event.module == "sfp_spyse" and (self.getTarget().targetValue in event.data):
return
domain_details = self.queryDomainDetails(eventData)
- if domain_details:
- domain_details_data = domain_details.get("data", {}).get("items", [])
- if len(domain_details_data) > 0:
- domain_item = domain_details_data[0]
- if domain_item.get("organizations"):
- for org in domain_item.get("organizations", []):
- if org.get("crunchbase"):
- if org.get("crunchbase").get("is_primary", False):
- org_name = org["crunchbase"].get("legal_name")
- if not org_name:
- org_name = org["crunchbase"].get("name")
- if org_name:
- evt = SpiderFootEvent('COMPANY_NAME', org_name, self.__name__, event)
- self.notifyListeners(evt)
-
- domain_dns = domain_item.get("dns_records")
- if domain_dns:
- domain_dns_a_records = domain_dns.get("A")
- if domain_dns_a_records:
- for dns_A in domain_dns_a_records:
- evt = SpiderFootEvent('IP_ADDRESS', dns_A, self.__name__, event)
- self.notifyListeners(evt)
+ if not domain_details:
+ return
- domain_dns_aaaa_records = domain_dns.get("AAAA")
- if domain_dns_aaaa_records:
- for dns_AAAA in domain_dns_aaaa_records:
- evt = SpiderFootEvent('IPV6_ADDRESS', dns_AAAA, self.__name__, event)
- self.notifyListeners(evt)
+ domain_details_data = domain_details.get("data", {}).get("items", [])
+ if len(domain_details_data) == 0:
+ return
- domain_dns_spf_records = domain_dns.get("SPF")
- if domain_dns_spf_records:
- for dns_spf in domain_dns_spf_records:
- if dns_spf.get("raw"):
- evt = SpiderFootEvent('DNS_SPF', dns_spf["raw"], self.__name__, event)
- self.notifyListeners(evt)
-
- domain_dns_txt_records = domain_dns.get("TXT")
- if domain_dns_txt_records:
- for dns_txt in domain_dns_txt_records:
- evt = SpiderFootEvent('DNS_TEXT', dns_txt, self.__name__, event)
+ domain_item = domain_details_data[0]
+ if domain_item.get("organizations"):
+ for org in domain_item.get("organizations", []):
+ crunchbase = org.get('crunchbase')
+ if crunchbase:
+ if crunchbase.get("is_primary", False):
+ org_name = crunchbase.get("legal_name")
+ if not org_name:
+ org_name = crunchbase.get("name")
+ if org_name:
+ evt = SpiderFootEvent('COMPANY_NAME', org_name, self.__name__, event)
self.notifyListeners(evt)
- domain_dns_ns_records = domain_dns.get("NS")
- if domain_dns_ns_records:
- for dns_ns in domain_dns_ns_records:
- evt = SpiderFootEvent('PROVIDER_DNS', dns_ns, self.__name__, event)
- self.notifyListeners(evt)
+ domain_dns = domain_item.get("dns_records")
+ if domain_dns:
+ domain_dns_a_records = domain_dns.get("A")
+ if domain_dns_a_records:
+ for dns_A in domain_dns_a_records:
+ if dns_A:
+ evt = SpiderFootEvent('IP_ADDRESS', dns_A, self.__name__, event)
+ self.notifyListeners(evt)
- domain_dns_mx_records = domain_dns.get("MX")
- if domain_dns_mx_records:
- for dns_mx in domain_dns_mx_records:
- evt = SpiderFootEvent('PROVIDER_MAIL', dns_mx, self.__name__, event)
- self.notifyListeners(evt)
+ domain_dns_aaaa_records = domain_dns.get("AAAA")
+ if domain_dns_aaaa_records:
+ for dns_AAAA in domain_dns_aaaa_records:
+ if dns_AAAA:
+ evt = SpiderFootEvent('IPV6_ADDRESS', dns_AAAA, self.__name__, event)
+ self.notifyListeners(evt)
- hosts_enrichment = domain_item.get("hosts_enrichment")
- if hosts_enrichment:
- for host_enrichment in hosts_enrichment:
- if host_enrichment.get("country"):
- evt = SpiderFootEvent('COUNTRY_NAME', host_enrichment["country"], self.__name__, event)
+ domain_dns_spf_records = domain_dns.get("SPF")
+ if domain_dns_spf_records:
+ for dns_spf in domain_dns_spf_records:
+ if dns_spf:
+ dns_spf_raw = dns_spf.get("raw")
+ if dns_spf_raw:
+ evt = SpiderFootEvent('DNS_SPF', dns_spf_raw, self.__name__, event)
self.notifyListeners(evt)
- domain_technologies = domain_item.get("technologies")
- if domain_technologies:
- for tech in domain_technologies:
- if tech.get("version"):
- evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', f"{tech['name']} {tech['version']}",
- self.__name__, event)
- else:
- evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', tech['name'], self.__name__, event)
+ domain_dns_txt_records = domain_dns.get("TXT")
+ if domain_dns_txt_records:
+ for dns_txt in domain_dns_txt_records:
+ if dns_txt:
+ evt = SpiderFootEvent('DNS_TEXT', dns_txt, self.__name__, event)
+ self.notifyListeners(evt)
+ domain_dns_ns_records = domain_dns.get("NS")
+ if domain_dns_ns_records:
+ for dns_ns in domain_dns_ns_records:
+ if dns_ns:
+ evt = SpiderFootEvent('PROVIDER_DNS', dns_ns, self.__name__, event)
self.notifyListeners(evt)
- domain_cves = domain_item.get("cve_list")
- if domain_cves:
- for cve in domain_cves:
- evt = SpiderFootEvent('VULNERABILITY', cve["id"], self.__name__, event)
+ domain_dns_mx_records = domain_dns.get("MX")
+ if domain_dns_mx_records:
+ for dns_mx in domain_dns_mx_records:
+ if dns_mx:
+ evt = SpiderFootEvent('PROVIDER_MAIL', dns_mx, self.__name__, event)
self.notifyListeners(evt)
- domain_whois = domain_item.get("whois_parsed")
- if domain_whois:
- domain_whois_registrar = domain_whois.get("registrar")
- if domain_whois_registrar:
- if domain_whois_registrar.get("registrar_name"):
- evt = SpiderFootEvent('DOMAIN_REGISTRAR',
- domain_whois_registrar["registrar_name"], self.__name__,
- event)
- self.notifyListeners(evt)
+ hosts_enrichment = domain_item.get("hosts_enrichment")
+ if hosts_enrichment:
+ for host_enrichment in hosts_enrichment:
+ city = host_enrichment.get("city")
+ country = host_enrichment.get("country")
+ location = ', '.join([_f for _f in [city, country] if _f])
+ if location:
+ evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)
+ self.notifyListeners(evt)
- domain_http_extract = domain_item.get("http_extract")
- if domain_http_extract:
+ domain_technologies = domain_item.get("technologies")
+ if domain_technologies:
+ for tech in domain_technologies:
+ name = tech.get("name")
- if domain_http_extract.get("http_status_code"):
- evt = SpiderFootEvent('HTTP_CODE', str(domain_http_extract["http_status_code"]),
- self.__name__, event)
- self.notifyListeners(evt)
+ if not name:
+ continue
- domain_emails = domain_http_extract.get("emails")
- if domain_emails:
- for email in domain_emails:
- evt = SpiderFootEvent('EMAILADDR', email, self.__name__, event)
- self.notifyListeners(evt)
+ version = tech.get("version")
+ software = ' '.join(filter(None, [name, version]))
+ if software:
+ evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', f"{software}", self.__name__, event)
+ self.notifyListeners(evt)
- domain_cert_summary = domain_item.get("cert_summary")
- if domain_cert_summary:
- domain_cert_summary_subject = domain_cert_summary.get("subject")
- if domain_cert_summary_subject:
- if domain_cert_summary_subject.get("organization"):
- evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED',
- domain_cert_summary_subject["organization"], self.__name__,
- event)
- self.notifyListeners(evt)
+ domain_cves = domain_item.get("cve_list")
+ if domain_cves:
+ for cve in domain_cves:
+ cve_id = cve.get('id')
+ if cve_id:
+ evt = SpiderFootEvent('VULNERABILITY', cve_id, self.__name__, event)
+ self.notifyListeners(evt)
- domain_cert_summary_issuer = domain_cert_summary.get("issuer")
- if domain_cert_summary_issuer:
- if domain_cert_summary_issuer.get("organization"):
- evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER',
- domain_cert_summary_issuer["organization"], self.__name__,
- event)
- self.notifyListeners(evt)
+ domain_whois = domain_item.get("whois_parsed")
+ if domain_whois:
+ domain_whois_registrar = domain_whois.get("registrar")
+ if domain_whois_registrar:
+ registrar_name = domain_whois_registrar.get("registrar_name")
+ if registrar_name:
+ evt = SpiderFootEvent('DOMAIN_REGISTRAR', registrar_name, self.__name__, event)
+ self.notifyListeners(evt)
+
+ domain_http_extract = domain_item.get("http_extract")
+ if domain_http_extract:
+ http_status_code = domain_http_extract.get("http_status_code")
+ if http_status_code:
+ evt = SpiderFootEvent('HTTP_CODE', str(http_status_code), self.__name__, event)
+ self.notifyListeners(evt)
- domain_trackers = domain_item.get("trackers")
- if domain_trackers:
- if domain_trackers.get("google_analytics_key"):
- evt = SpiderFootEvent("WEB_ANALYTICS_ID",
- f"Google Analytics: {domain_trackers.get('google_analytics_key')}",
- self.__name__, event)
+ http_headers = domain_http_extract.get("http_headers")
+ if http_headers:
+ evt = SpiderFootEvent('WEBSERVER_HTTPHEADERS', str(http_headers), self.__name__, event)
+ self.notifyListeners(evt)
+
+ domain_emails = domain_http_extract.get("emails")
+ if domain_emails:
+ for email in domain_emails:
+ if self.sf.validEmail(email):
+ evt = SpiderFootEvent('EMAILADDR', email, self.__name__, event)
self.notifyListeners(evt)
- while nextPageHasData:
+ domain_cert_summary = domain_item.get("cert_summary")
+ if domain_cert_summary:
+ domain_cert_summary_subject = domain_cert_summary.get("subject")
+ if domain_cert_summary_subject:
+ cert_issued = domain_cert_summary_subject.get("organization")
+ if cert_issued:
+ evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert_issued, self.__name__, event)
+ self.notifyListeners(evt)
+
+ domain_cert_summary_issuer = domain_cert_summary.get("issuer")
+ if domain_cert_summary_issuer:
+ cert_issuer = domain_cert_summary_issuer.get("organization")
+ if cert_issuer:
+ evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert_issuer, self.__name__, event)
+ self.notifyListeners(evt)
+
+ domain_trackers = domain_item.get("trackers")
+ if domain_trackers:
+ google_analytics_key = domain_trackers.get("google_analytics_key")
+ if google_analytics_key:
+ evt = SpiderFootEvent("WEB_ANALYTICS_ID", f"Google Analytics: {google_analytics_key}", self.__name__, event)
+ self.notifyListeners(evt)
+
+ def retrieve_subdomains(self, event):
+ if event.module == "sfp_spyse" and (self.getTarget().targetValue in event.data):
+ return
+
+ eventData = event.data
+ domains = list()
+ currentOffset = 0
+ while True:
if self.checkForStop():
- return
+ break
+
+ if self.errorState:
+ break
data = self.querySubdomains(eventData, currentOffset)
if not data:
- nextPageHasData = False
break
data = data.get("data")
if data is None:
- self.debug("No subdomains found for domain " + eventData)
- nextPageHasData = False
+ self.debug(f"No subdomains found for domain {eventData}")
break
- else:
- records = data.get('items')
- if records:
- for record in records:
- domain = record.get('name')
- if domain:
- evt = SpiderFootEvent('RAW_RIR_DATA', str(record), self.__name__, event)
- self.notifyListeners(evt)
- domains.append(domain)
- self.reportExtraData(record, event)
+ records = data.get('items')
+ if records:
+ for record in records:
+ print(record)
+ domain = record.get('name')
+ if domain:
+ evt = SpiderFootEvent('RAW_RIR_DATA', str(record), self.__name__, event)
+ self.notifyListeners(evt)
+ domains.append(domain)
# Calculate if there are any records in the next offset (page)
if len(records) < self.limit:
- nextPageHasData = False
+ break
+
currentOffset += self.limit
for domain in set(domains):
-
if domain in self.results:
continue