Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Split sfp_dns into smaller modules.

Stevea450259e5e68
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.py
new file mode 100644
index 00000000..ff02e2fa
--- /dev/null
+++ b/modules/sfp_dnsbrute.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_dnsbrute
+# Purpose: SpiderFoot plug-in for attempting to resolve through brute-forcing
+# common hostnames.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 06/07/2017
+# Copyright: (c) Steve Micallef 2017
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import socket
+import re
+import dns
+import urllib2
+from netaddr import IPAddress, IPNetwork
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_dnsbrute(SpiderFootPlugin):
+ """DNS Brute-force:Footprint,Investigate:DNS::Attempts to identify hostnames through brute-forcing common names."""
+
+
+ # Default options
+ opts = {
+ "skipcommonwildcard": True,
+ "domainonly": True,
+ "commonsubs": ["www", "web", "ns", "mail", "dns", "mx", "gw", "proxy",
+ "ssl", "fw", "gateway", "firewall", "www1", "www2",
+ "ns0", "ns1", "ns2", "dns0", "dns1", "dns2", "mx1", "mx2"
+ ] # Common sub-domains to try.
+
+ }
+
+ # Option descriptions
+ optdescs = {
+ 'skipcommonwildcard': "If wildcard DNS is detected, only attempt to look up the first common sub-domain from the common sub-domain list.",
+ 'domainonly': "Only attempt to brute-force names on domain names, not hostnames (some hostnames are also sub-domains).",
+ 'commonsubs': "Common sub-domains to try to resolve on the target subdomain/domain. Prefix with an '@' to iterate through a file containing sub-domains to try (one per line), e.g. @C:\subdomains.txt or @/home/bob/subdomains.txt. Or supply a URL to load the list from there."
+ }
+
+ events = dict()
+ resolveCache = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.events = dict()
+ self.resolveCache = dict()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ self.sublist = self.opts['commonsubs']
+ # User may have supplied a file or URL containing the subdomains
+ if self.opts['commonsubs'][0].startswith("http://") or \
+ self.opts['commonsubs'][0].startswith("https://") or \
+ self.opts['commonsubs'][0].startswith("@"):
+ self.sublist = self.sf.optValueToData(self.opts['commonsubs'][0])
+
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ['INTERNET_NAME', 'DOMAIN_NAME']
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["IP_ADDRESS", "INTERNET_NAME", "IPV6_ADDRESS"]
+
+ # Resolve a host
+ def resolveHost(self, hostname):
+ if hostname in self.resolveCache:
+ self.sf.debug("Returning cached result for " + hostname + " (" +
+ str(self.resolveCache[hostname]) + ")")
+ return self.resolveCache[hostname]
+
+ try:
+ addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))
+ self.resolveCache[hostname] = addrs
+ self.sf.debug("Resolved " + hostname + " to: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")
+ return list()
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+ eventDataHash = self.sf.hashstring(eventData)
+ parentEvent = event
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if self.opts['domainonly'] and eventName == "INTERNET_NAME":
+ return None
+
+ if eventDataHash in self.events:
+ return None
+
+ self.events[eventDataHash] = True
+
+ self.sf.debug("Iterating through possible sub-domains.")
+ wildcard = self.sf.checkDnsWildcard(eventData)
+ if self.opts['skipcommonwildcard'] and wildcard:
+ self.sf.debug("Wildcard DNS detected.")
+ return None
+
+ # Try resolving common names
+ for sub in self.sublist:
+ if self.checkForStop():
+ return None
+
+ if sub == "":
+ continue
+ else:
+ name = sub + "." + eventData
+
+ # Skip hosts we've processed already
+ if self.sf.hashstring(name) in self.events.keys():
+ continue
+
+ if len(self.resolveHost(name)) > 0:
+ # Report the host
+ evt = SpiderFootEvent("INTERNET_NAME", name,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+
+# End of sfp_dnsbrute class
diff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.py
new file mode 100644
index 00000000..383bab15
--- /dev/null
+++ b/modules/sfp_dnsneighbor.py
@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_dnsneighbor
+# Purpose: SpiderFoot plug-in for gathering IP addresses from sub-domains
+# and hostnames identified, and optionally affiliates.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 07/07/2017
+# Copyright: (c) Steve Micallef 2017
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import socket
+import re
+import dns
+import urllib2
+from netaddr import IPAddress, IPNetwork
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_dnsneighbor(SpiderFootPlugin):
+ """DNS Look-aside:Footprint,Investigate:DNS::Attempt to reverse-resolve the IP addresses next to your target to see if they are related."""
+
+ # Default options
+ opts = {
+ 'lookasidecount': 10,
+ 'validatereverse': True
+ }
+
+ # Option descriptions
+ optdescs = {
+ 'validatereverse': "Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.",
+ 'lookasidecount': "If look-aside is enabled, the number of IPs on each 'side' of the IP to look up"
+ }
+
+ events = dict()
+ domresults = dict()
+ hostresults = dict()
+ resolveCache = dict()
+ resolveCache6 = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.events = dict()
+ self.domresults = dict()
+ self.hostresults = dict()
+ self.resolveCache = dict()
+ self.resolveCache6 = dict()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ['IP_ADDRESS']
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["INTERNET_NAME"]
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+ eventDataHash = self.sf.hashstring(eventData)
+ addrs = None
+ parentEvent = event
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if eventDataHash in self.events:
+ return None
+
+ self.events[eventDataHash] = True
+
+ try:
+ ip = IPAddress(eventData)
+ except BaseException as e:
+ self.sf.error("Invalid IP address received: " + str(ip), False)
+ return None
+
+ minip = IPAddress(int(ip) - self.opts['lookasidecount'])
+ maxip = IPAddress(int(ip) + self.opts['lookasidecount'])
+ self.sf.debug("Lookaside max: " + str(maxip) + ", min: " + str(minip))
+ s = int(minip)
+ c = int(maxip)
+
+ while s <= c:
+ sip = str(IPAddress(s))
+ self.sf.debug("Attempting look-aside lookup of: " + sip)
+ if self.checkForStop():
+ return None
+
+ if sip in self.hostresults or sip == eventData:
+ s += 1
+ continue
+
+ addrs = self.resolveIP(sip)
+ if len(addrs) == 0:
+ self.sf.debug("Look-aside resolve for " + sip + " failed.")
+ s += 1
+ continue
+
+ # Report addresses that resolve to hostnames on the same
+ # domain or sub-domain as the target.
+ if self.getTarget().matches(sip):
+ affil = False
+ else:
+ affil = True
+ for a in addrs:
+ if self.getTarget().matches(a):
+ affil = False
+
+ # Generate the event for the look-aside IP, but don't let it re-trigger
+ # this module by adding it to self.events first.
+ self.events[sip] = True
+ ev = self.processHost(sip, parentEvent, affil)
+
+ for addr in addrs:
+ if self.checkForStop():
+ return None
+
+ if addr == sip:
+ continue
+ if self.sf.validIP(addr):
+ parent = parentEvent
+ else:
+ # Hostnames from the IP need to be linked to the IP
+ parent = ev
+
+ if self.getTarget().matches(addr):
+ # Generate an event for the IP, then
+ # let the handling by this module take
+ # care of follow-up processing.
+ self.processHost(addr, parent, False)
+ else:
+ self.processHost(addr, parent, True)
+ s += 1
+
+ # Resolve an IP
+ def resolveIP(self, ipaddr):
+ ret = list()
+ self.sf.debug("Performing reverse-resolve of " + ipaddr)
+
+ if ipaddr in self.resolveCache:
+ self.sf.debug("Returning cached result for " + ipaddr + " (" +
+ str(self.resolveCache[ipaddr]) + ")")
+ return self.resolveCache[ipaddr]
+
+ try:
+ addrs = self.sf.normalizeDNS(socket.gethostbyaddr(ipaddr))
+ self.resolveCache[ipaddr] = addrs
+ self.sf.debug("Resolved " + ipaddr + " to: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to resolve " + ipaddr + " (" + str(e) + ")")
+ self.resolveCache[ipaddr] = list()
+ return ret
+
+ # Resolve a host
+ def resolveHost(self, hostname):
+ if hostname in self.resolveCache:
+ self.sf.debug("Returning cached result for " + hostname + " (" +
+ str(self.resolveCache[hostname]) + ")")
+ return self.resolveCache[hostname]
+
+ try:
+ addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))
+ self.resolveCache[hostname] = addrs
+ self.sf.debug("Resolved " + hostname + " to: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")
+ return list()
+
+ # Resolve a host to IPv6
+ def resolveHost6(self, hostname):
+ if hostname in self.resolveCache6:
+ self.sf.debug("Returning IPv6 cached result for " + hostname + " (" +
+ str(self.resolveCache6[hostname]) + ")")
+ return self.resolveCache6[hostname]
+
+ try:
+ addrs = list()
+ res = socket.getaddrinfo(hostname, None, socket.AF_INET6)
+ for addr in res:
+ if addr[4][0] not in addrs:
+ addrs.append(addr[4][0])
+ if len(addrs) < 1:
+ return None
+ self.resolveCache6[hostname] = addrs
+ self.sf.debug("Resolved " + hostname + " to IPv6: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to IPv6 resolve " + hostname + " (" + str(e) + ")")
+ return list()
+
+ # Process a host/IP, parentEvent is the event that represents this entity
+ def processHost(self, host, parentEvent, affiliate=None):
+ parentHash = self.sf.hashstring(parentEvent.data)
+ if host not in self.hostresults:
+ self.hostresults[host] = list(parentHash)
+ else:
+ if parentHash in self.hostresults[host] or parentEvent.data == host:
+ self.sf.debug("Skipping host, " + host + ", already processed.")
+ return None
+ else:
+ self.hostresults[host].append(parentHash)
+
+ self.sf.debug("Found host: " + host)
+ # If the returned hostname is aliaseed to our
+ # target in some way, flag it as an affiliate
+ if affiliate is None:
+ affil = True
+ if self.getTarget().matches(host):
+ affil = False
+ # If the IP the host resolves to is in our
+ # list of aliases,
+ if not self.sf.validIP(host):
+ for hostip in self.resolveHost(host):
+ if self.getTarget().matches(hostip):
+ affil = False
+ else:
+ affil = affiliate
+
+ if affil:
+ if self.sf.validIP(host):
+ htype = "AFFILIATE_IPADDR"
+ else:
+ htype = "AFFILIATE_INTERNET_NAME"
+ else:
+ if self.sf.validIP(host):
+ htype = "IP_ADDRESS"
+ else:
+ htype = "INTERNET_NAME"
+
+ # Report the host
+ evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)
+ self.notifyListeners(evt)
+
+ return evt
+
+# End of sfp_dnsneighbor class
diff --git a/modules/sfp_dnsraw.py b/modules/sfp_dnsraw.py
new file mode 100644
index 00000000..271260ef
--- /dev/null
+++ b/modules/sfp_dnsraw.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_dnsraw
+# Purpose: SpiderFoot plug-in for collecting raw DNS records.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 07/07/2017
+# Copyright: (c) Steve Micallef 2017
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import socket
+import re
+import dns
+import urllib2
+from netaddr import IPAddress, IPNetwork
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_dnsraw(SpiderFootPlugin):
+ """DNS Raw Records:Footprint,Investigate:DNS::Retrieves raw DNS records such as MX, TXT and others."""
+
+ # Default options
+ opts = {
+ }
+
+ # Option descriptions
+ optdescs = {
+ }
+
+ events = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.events = dict()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ['INTERNET_NAME', 'DOMAIN_NAME']
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["PROVIDER_MAIL", "PROVIDER_DNS", "RAW_DNS_RECORDS",
+ "DNS_TEXT", "DNS_SPF", "AFFILIATE_INTERNET_NAME"]
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+ eventDataHash = self.sf.hashstring(eventData)
+ addrs = None
+ parentEvent = event
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if eventDataHash in self.events:
+ self.sf.debug("Skipping duplicate event for " + eventData)
+ return None
+
+ self.events[eventDataHash] = True
+
+ self.sf.debug("Gathering DNS records for " + eventData)
+ # Process the raw data alone
+ recdata = dict()
+ recs = {
+ 'MX': ['\S+\s+(?:\d+)?\s+IN\s+MX\s+\d+\s+(\S+)\.', 'PROVIDER_MAIL'],
+ 'NS': ['\S+\s+(?:\d+)?\s+IN\s+NS\s+(\S+)\.', 'PROVIDER_DNS'],
+ 'TXT': ['\S+\s+TXT\s+\"(.[^\"]*)"', 'DNS_TEXT']
+ }
+
+ for rec in recs.keys():
+ if self.checkForStop():
+ return None
+
+ try:
+ req = dns.message.make_query(eventData, dns.rdatatype.from_text(rec))
+
+ if self.opts['_dnsserver'] != "":
+ n = self.opts['_dnsserver']
+ else:
+ ns = dns.resolver.get_default_resolver()
+ n = ns.nameservers[0]
+
+ res = dns.query.udp(req, n, timeout=30)
+ for x in res.answer:
+ for rx in recs.keys():
+ self.sf.debug("Checking " + str(x) + " + against " + recs[rx][0])
+ pat = re.compile(recs[rx][0], re.IGNORECASE | re.DOTALL)
+ grps = re.findall(pat, str(x))
+ if len(grps) > 0:
+ for m in grps:
+ self.sf.debug("Matched: " + m)
+ strdata = unicode(m, 'utf-8', errors='replace')
+ evt = SpiderFootEvent(recs[rx][1], strdata,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ if rec != "TXT" and not strdata.endswith(eventData):
+ evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME",
+ strdata, self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ if rec == "TXT" and "v=spf" in strdata:
+ evt = SpiderFootEvent("DNS_SPF", strdata,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+
+ else:
+ strdata = unicode(str(x), 'utf-8', errors='replace')
+ evt = SpiderFootEvent("RAW_DNS_RECORDS", strdata,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ except BaseException as e:
+ self.sf.error("Failed to obtain DNS response for " + eventData +
+ "(" + rec + "): " + str(e), False)
+
+# End of sfp_dnsraw class
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
new file mode 100644
index 00000000..9349c7cf
--- /dev/null
+++ b/modules/sfp_dnsresolve.py
@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_dnsresolve
+# Purpose: SpiderFoot plug-in for extracting hostnames from identified data
+# and resolving them.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 07/07/2017
+# Copyright: (c) Steve Micallef 2017
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import socket
+import re
+import dns
+import urllib2
+from netaddr import IPAddress, IPNetwork
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_dnsresolve(SpiderFootPlugin):
+ """DNS Resolver:Footprint,Investigate:DNS::Resolves Hosts and IP Addresses identified, also extracted from raw content."""
+
+ # Default options
+ opts = {
+ 'onlyactive': True,
+ 'validatereverse': True,
+ 'skipcommononwildcard': True,
+ 'netblocklookup': True,
+ 'maxnetblock': 24
+ }
+
+ # Option descriptions
+ optdescs = {
+ 'skipcommononwildcard': "If wildcard DNS is detected, only attempt to look up the first common sub-domain from the common sub-domain list.",
+ 'onlyactive': "Only report sub-domains/hostnames that resolve to an IP.",
+ 'validatereverse': "Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.",
+ 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?",
+ 'maxnetblock': "Maximum owned netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"
+ }
+
+ events = dict()
+ domresults = dict()
+ hostresults = dict()
+ resolveCache = dict()
+ resolveCache6 = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.events = dict()
+ self.domresults = dict()
+ self.hostresults = dict()
+ self.resolveCache = dict()
+ self.resolveCache6 = dict()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ def enrichTarget(self, target):
+ ret = list()
+ # If it's an IP, get the hostname it reverse resolves to
+ if target.getType() == "IP_ADDRESS":
+ ret = self.resolveIP(target.getValue())
+ if target.getType() == "INTERNET_NAME":
+ ret = self.resolveHost(target.getValue())
+ if target.getType() == "NETBLOCK_OWNER":
+ ret = list()
+ for addr in IPNetwork(target.getValue()):
+ ipaddr = str(addr)
+ if ipaddr.split(".")[3] in ['255', '0']:
+ continue
+ if '255' in ipaddr.split("."):
+ continue
+ ret.append(ipaddr)
+
+ # Add the reverse-resolved hostnames as aliases too..
+ names = self.resolveIP(ipaddr)
+
+ if self.opts['validatereverse']:
+ for host in names:
+ chk = self.resolveHost(host)
+ if ipaddr in chk:
+ ret.append(host)
+ else:
+ ret.extend(names)
+
+ for host in ret:
+ if self.sf.validIP(host):
+ target.setAlias(host, "IP_ADDRESS")
+ else:
+ target.setAlias(host, "INTERNET_NAME")
+ # If the target was a hostname/sub-domain, we can
+ # add the domain as an alias for the target. But
+ # not if the target was an IP or subnet.
+ #if target.getType() == "INTERNET_NAME":
+ # dom = self.sf.hostDomain(host, self.opts['_internettlds'])
+ # target.setAlias(dom, "INTERNET_NAME")
+
+ self.sf.debug("Aliases identified: " + str(target.getAliases()))
+
+ return target
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ['*']
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",
+ "AFFILIATE_IPADDR", "DOMAIN_NAME", "IPV6_ADDRESS",
+ "DOMAIN_NAME_PARENT"]
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+ eventDataHash = self.sf.hashstring(eventData)
+ addrs = None
+ parentEvent = event
+
+ # Don't be recursive
+ if srcModuleName == "sfp_dnsresolve" or eventName == "ROOT":
+ return None
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if eventDataHash in self.events:
+ self.sf.debug("Skipping duplicate event for " + eventData)
+ return None
+ self.events[eventDataHash] = True
+
+ # Search for IPs/hosts in raw data, but obviously nothing this module
+ # already produces, as those things are already entities, not raw data.
+ if eventName not in self.producedEvents():
+ data = urllib2.unquote(eventData)
+ for name in self.getTarget().getNames():
+ if self.checkForStop():
+ return None
+ pat = re.compile("(%..)?([a-zA-Z0-9\-\.]+\." + name + ")", re.IGNORECASE)
+ matches = re.findall(pat, data)
+ if matches is not None:
+ for match in matches:
+ self.processHost(match[1], parentEvent, False)
+ # Nothing left to do with internal links and raw data
+ return None
+
+ if eventName == 'NETBLOCK_OWNER' and self.opts['netblocklookup']:
+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:
+ self.sf.debug("Network size bigger than permitted: " +
+ str(IPNetwork(eventData).prefixlen) + " > " +
+ str(self.opts['maxnetblock']))
+ return None
+
+ # Not handling IPv6 (yet)
+ if "::" in eventData:
+ return None
+
+ self.sf.debug("Looking up IPs in owned netblock: " + eventData)
+ for ip in IPNetwork(eventData):
+ ipaddr = str(ip)
+ if "::" in ipaddr:
+ continue
+ if ipaddr.split(".")[3] in ['255', '0']:
+ continue
+ if '255' in ipaddr.split("."):
+ continue
+
+ if self.checkForStop():
+ return None
+ addrs = self.resolveIP(ipaddr)
+
+ if len(addrs) > 0:
+ self.sf.debug("Found a reversed hostname from " + ipaddr +
+ " (" + str(addrs) + ")")
+ for addr in addrs:
+ # Generate an event for the IP, then
+ # let the handling by this module take
+ # care of follow-up processing.
+ if self.checkForStop():
+ return None
+
+ self.processHost(ipaddr, parentEvent, False)
+ return None
+
+ if eventName in ["IP_ADDRESS", "INTERNET_NAME",
+ "AFFILIATE_IPADDR", "AFFILIATE_INTERNET_NAME"]:
+
+ if "_NAME" in eventName:
+ addrs = self.resolveHost(eventData)
+ else:
+ addrs = self.resolveIP(eventData)
+
+ # We now have a set of hosts/IPs to do something with.
+ for addr in addrs:
+ if self.checkForStop():
+ return None
+
+ # Does the host/IP match to the original target?
+ if self.getTarget().matches(addr):
+ self.processHost(addr, parentEvent, False)
+ else:
+ # IP addresses resolved from hosts are assumed
+ # to be part of the target (non-affiliates).
+ if self.sf.validIP(addr) and "AFFILIATE_" not in eventName:
+ self.processHost(addr, parentEvent, False)
+ else:
+ self.processHost(addr, parentEvent, True)
+
+ # Resolve an IP
+ def resolveIP(self, ipaddr):
+ ret = list()
+ self.sf.debug("Performing reverse-resolve of " + ipaddr)
+
+ if ipaddr in self.resolveCache:
+ self.sf.debug("Returning cached result for " + ipaddr + " (" +
+ str(self.resolveCache[ipaddr]) + ")")
+ return self.resolveCache[ipaddr]
+
+ try:
+ addrs = self.sf.normalizeDNS(socket.gethostbyaddr(ipaddr))
+ self.resolveCache[ipaddr] = addrs
+ self.sf.debug("Resolved " + ipaddr + " to: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to resolve " + ipaddr + " (" + str(e) + ")")
+ self.resolveCache[ipaddr] = list()
+ return ret
+
+ # Resolve a host
+ def resolveHost(self, hostname):
+ if hostname in self.resolveCache:
+ self.sf.debug("Returning cached result for " + hostname + " (" +
+ str(self.resolveCache[hostname]) + ")")
+ return self.resolveCache[hostname]
+
+ try:
+ addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))
+ self.resolveCache[hostname] = addrs
+ self.sf.debug("Resolved " + hostname + " to: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")
+ return list()
+
+ # Resolve a host to IPv6
+ def resolveHost6(self, hostname):
+ if hostname in self.resolveCache6:
+ self.sf.debug("Returning IPv6 cached result for " + hostname + " (" +
+ str(self.resolveCache6[hostname]) + ")")
+ return self.resolveCache6[hostname]
+
+ try:
+ addrs = list()
+ res = socket.getaddrinfo(hostname, None, socket.AF_INET6)
+ for addr in res:
+ if addr[4][0] not in addrs:
+ addrs.append(addr[4][0])
+ if len(addrs) < 1:
+ return None
+ self.resolveCache6[hostname] = addrs
+ self.sf.debug("Resolved " + hostname + " to IPv6: " + str(addrs))
+ return addrs
+ except BaseException as e:
+ self.sf.debug("Unable to IPv6 resolve " + hostname + " (" + str(e) + ")")
+ return list()
+
+ # Process a host/IP, parentEvent is the event that represents this entity
+ def processHost(self, host, parentEvent, affiliate=None):
+ parentHash = self.sf.hashstring(parentEvent.data)
+ if host not in self.hostresults:
+ self.hostresults[host] = list(parentHash)
+ else:
+ if parentHash in self.hostresults[host] or parentEvent.data == host:
+ self.sf.debug("Skipping host, " + host + ", already processed.")
+ return None
+ else:
+ self.hostresults[host].append(parentHash)
+
+ self.sf.debug("Found host: " + host)
+ # If the returned hostname is aliaseed to our
+ # target in some way, flag it as an affiliate
+ if affiliate is None:
+ affil = True
+ if self.getTarget().matches(host):
+ affil = False
+ # If the IP the host resolves to is in our
+ # list of aliases,
+ if not self.sf.validIP(host):
+ for hostip in self.resolveHost(host):
+ if self.getTarget().matches(hostip):
+ affil = False
+ else:
+ affil = affiliate
+
+ if affil:
+ if self.sf.validIP(host):
+ htype = "AFFILIATE_IPADDR"
+ else:
+ htype = "AFFILIATE_INTERNET_NAME"
+ else:
+ if self.sf.validIP(host):
+ htype = "IP_ADDRESS"
+ else:
+ htype = "INTERNET_NAME"
+
+ if htype.endswith("INTERNET_NAME") and self.opts['onlyactive']:
+ if len(self.resolveHost(host)) == 0:
+ return None
+
+ # Report the host
+ evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ # Report the domain for that host
+ if htype == "INTERNET_NAME":
+ dom = self.sf.hostDomain(host, self.opts['_internettlds'])
+ self.processDomain(dom, evt)
+
+ # Try obtain the IPv6 address
+ for ip6 in self.resolveHost6(host):
+ evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt)
+ self.notifyListeners(evt6)
+
+ return evt
+
+ def processDomain(self, domainName, parentEvent):
+ if domainName not in self.domresults:
+ self.domresults[domainName] = True
+ else:
+ self.sf.debug("Skipping domain, " + domainName + ", already processed.")
+ return None
+
+ if self.getTarget().matches(domainName):
+ domevt = SpiderFootEvent("DOMAIN_NAME", domainName,
+ self.__name__, parentEvent)
+ self.notifyListeners(domevt)
+ else:
+ domevt = SpiderFootEvent("DOMAIN_NAME_PARENT", domainName,
+ self.__name__, parentEvent)
+ self.notifyListeners(domevt)
+ return None
+
+# End of sfp_dnsresolve class
diff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.py
new file mode 100644
index 00000000..18e136ce
--- /dev/null
+++ b/modules/sfp_dnszonexfer.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_dnszonexfer
+# Purpose: SpiderFoot plug-in to attempt a DNS zone transfer.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 08/07/2017
+# Copyright: (c) Steve Micallef 2017
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import socket
+import re
+import dns
+import urllib2
+from netaddr import IPAddress, IPNetwork
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_dnszonexfer(SpiderFootPlugin):
+ """DNS Zone Transfer:Footprint,Investigate:DNS::Attempts to perform a DNS zone transfer for domains."""
+
+ # Default options
+ opts = {
+ }
+
+ # Option descriptions
+ optdescs = {
+ }
+
+ events = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.events = dict()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ['DOMAIN_NAME']
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["RAW_DNS_RECORDS"]
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+ eventDataHash = self.sf.hashstring(eventData)
+ addrs = None
+ parentEvent = event
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if eventDataHash in self.events:
+ self.sf.debug("Skipping duplicate event for " + eventData)
+ return None
+
+ self.events[eventDataHash] = True
+
+ self.sf.debug("Gathering DNS records for " + eventData)
+ # Process the raw data alone
+ recdata = dict()
+ recs = {
+ 'MX': ['\S+\s+(?:\d+)?\s+IN\s+MX\s+\d+\s+(\S+)\.', 'PROVIDER_MAIL'],
+ 'NS': ['\S+\s+(?:\d+)?\s+IN\s+NS\s+(\S+)\.', 'PROVIDER_DNS'],
+ 'TXT': ['\S+\s+TXT\s+\"(.[^\"]*)"', 'DNS_TEXT']
+ }
+
+ for rec in recs.keys():
+ if self.checkForStop():
+ return None
+
+ try:
+ req = dns.message.make_query(eventData, dns.rdatatype.from_text(rec))
+
+ if self.opts['_dnsserver'] != "":
+ n = self.opts['_dnsserver']
+ else:
+ ns = dns.resolver.get_default_resolver()
+ n = ns.nameservers[0]
+
+ res = dns.query.udp(req, n, timeout=30)
+ for x in res.answer:
+ for rx in recs.keys():
+ self.sf.debug("Checking " + str(x) + " + against " + recs[rx][0])
+ pat = re.compile(recs[rx][0], re.IGNORECASE | re.DOTALL)
+ grps = re.findall(pat, str(x))
+ if len(grps) > 0:
+ for m in grps:
+ self.sf.debug("Matched: " + m)
+ strdata = unicode(m, 'utf-8', errors='replace')
+ evt = SpiderFootEvent(recs[rx][1], strdata,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ if rec != "TXT" and not strdata.endswith(eventData):
+ evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME",
+ strdata, self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ if rec == "TXT" and "v=spf" in strdata:
+ evt = SpiderFootEvent("DNS_SPF", strdata,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+
+ else:
+ strdata = unicode(str(x), 'utf-8', errors='replace')
+ evt = SpiderFootEvent("RAW_DNS_RECORDS", strdata,
+ self.__name__, parentEvent)
+ self.notifyListeners(evt)
+ except BaseException as e:
+ self.sf.error("Failed to obtain DNS response for " + eventData +
+ "(" + rec + "): " + str(e), False)
+
+# End of sfp_dnszonexfer class