SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Split sfp_dns into smaller modules.
Stevea450259e5e68
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.pynew file mode 100644index 00000000..ff02e2fa--- /dev/null+++ b/modules/sfp_dnsbrute.py@@ -0,0 +1,132 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_dnsbrute+# Purpose: SpiderFoot plug-in for attempting to resolve through brute-forcing+# common hostnames.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 06/07/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import socket+import re+import dns+import urllib2+from netaddr import IPAddress, IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_dnsbrute(SpiderFootPlugin):+ """DNS Brute-force:Footprint,Investigate:DNS::Attempts to identify hostnames through brute-forcing common names."""+++ # Default options+ opts = {+ "skipcommonwildcard": True,+ "domainonly": True,+ "commonsubs": ["www", "web", "ns", "mail", "dns", "mx", "gw", "proxy",+ "ssl", "fw", "gateway", "firewall", "www1", "www2",+ "ns0", "ns1", "ns2", "dns0", "dns1", "dns2", "mx1", "mx2"+ ] # Common sub-domains to try.++ }++ # Option descriptions+ optdescs = {+ 'skipcommonwildcard': "If wildcard DNS is detected, only attempt to look up the first common sub-domain from the common sub-domain list.",+ 'domainonly': "Only attempt to brute-force names on domain names, not hostnames (some hostnames are also sub-domains).",+ 'commonsubs': "Common sub-domains to try to resolve on the target subdomain/domain. Prefix with an '@' to iterate through a file containing sub-domains to try (one per line), e.g. @C:\subdomains.txt or @/home/bob/subdomains.txt. Or supply a URL to load the list from there."+ }++ events = dict()+ resolveCache = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.events = dict()+ self.resolveCache = dict()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ self.sublist = self.opts['commonsubs']+ # User may have supplied a file or URL containing the subdomains+ if self.opts['commonsubs'][0].startswith("http://") or \+ self.opts['commonsubs'][0].startswith("https://") or \+ self.opts['commonsubs'][0].startswith("@"):+ self.sublist = self.sf.optValueToData(self.opts['commonsubs'][0])+++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['INTERNET_NAME', 'DOMAIN_NAME']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["IP_ADDRESS", "INTERNET_NAME", "IPV6_ADDRESS"]++ # Resolve a host+ def resolveHost(self, hostname):+ if hostname in self.resolveCache:+ self.sf.debug("Returning cached result for " + hostname + " (" ++ str(self.resolveCache[hostname]) + ")")+ return self.resolveCache[hostname]++ try:+ addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))+ self.resolveCache[hostname] = addrs+ self.sf.debug("Resolved " + hostname + " to: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")+ return list()++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data+ eventDataHash = self.sf.hashstring(eventData)+ parentEvent = event++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['domainonly'] and eventName == "INTERNET_NAME":+ return None++ if eventDataHash in self.events:+ return None++ self.events[eventDataHash] = True++ self.sf.debug("Iterating through possible sub-domains.")+ wildcard = self.sf.checkDnsWildcard(eventData)+ if self.opts['skipcommonwildcard'] and wildcard:+ self.sf.debug("Wildcard DNS detected.")+ return None++ # Try resolving common names+ for sub in self.sublist:+ if self.checkForStop():+ return None++ if sub == "":+ continue+ else:+ name = sub + "." + eventData++ # Skip hosts we've processed already+ if self.sf.hashstring(name) in self.events.keys():+ continue++ if len(self.resolveHost(name)) > 0:+ # Report the host+ evt = SpiderFootEvent("INTERNET_NAME", name,+ self.__name__, parentEvent)+ self.notifyListeners(evt)++# End of sfp_dnsbrute classdiff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.pynew file mode 100644index 00000000..383bab15--- /dev/null+++ b/modules/sfp_dnsneighbor.py@@ -0,0 +1,246 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_dnsneighbor+# Purpose: SpiderFoot plug-in for gathering IP addresses from sub-domains+# and hostnames identified, and optionally affiliates.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 07/07/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import socket+import re+import dns+import urllib2+from netaddr import IPAddress, IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_dnsneighbor(SpiderFootPlugin):+ """DNS Look-aside:Footprint,Investigate:DNS::Attempt to reverse-resolve the IP addresses next to your target to see if they are related."""++ # Default options+ opts = {+ 'lookasidecount': 10,+ 'validatereverse': True+ }++ # Option descriptions+ optdescs = {+ 'validatereverse': "Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.",+ 'lookasidecount': "If look-aside is enabled, the number of IPs on each 'side' of the IP to look up"+ }++ events = dict()+ domresults = dict()+ hostresults = dict()+ resolveCache = dict()+ resolveCache6 = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.events = dict()+ self.domresults = dict()+ self.hostresults = dict()+ self.resolveCache = dict()+ self.resolveCache6 = dict()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['IP_ADDRESS']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["INTERNET_NAME"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data+ eventDataHash = self.sf.hashstring(eventData)+ addrs = None+ parentEvent = event++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventDataHash in self.events:+ return None++ self.events[eventDataHash] = True++ try:+ ip = IPAddress(eventData)+ except BaseException as e:+ self.sf.error("Invalid IP address received: " + str(ip), False)+ return None++ minip = IPAddress(int(ip) - self.opts['lookasidecount'])+ maxip = IPAddress(int(ip) + self.opts['lookasidecount'])+ self.sf.debug("Lookaside max: " + str(maxip) + ", min: " + str(minip))+ s = int(minip)+ c = int(maxip)++ while s <= c:+ sip = str(IPAddress(s))+ self.sf.debug("Attempting look-aside lookup of: " + sip)+ if self.checkForStop():+ return None++ if sip in self.hostresults or sip == eventData:+ s += 1+ continue++ addrs = self.resolveIP(sip)+ if len(addrs) == 0:+ self.sf.debug("Look-aside resolve for " + sip + " failed.")+ s += 1+ continue++ # Report addresses that resolve to hostnames on the same+ # domain or sub-domain as the target.+ if self.getTarget().matches(sip):+ affil = False+ else:+ affil = True+ for a in addrs:+ if self.getTarget().matches(a):+ affil = False++ # Generate the event for the look-aside IP, but don't let it re-trigger+ # this module by adding it to self.events first.+ self.events[sip] = True+ ev = self.processHost(sip, parentEvent, affil)++ for addr in addrs:+ if self.checkForStop():+ return None++ if addr == sip:+ continue+ if self.sf.validIP(addr):+ parent = parentEvent+ else:+ # Hostnames from the IP need to be linked to the IP+ parent = ev++ if self.getTarget().matches(addr):+ # Generate an event for the IP, then+ # let the handling by this module take+ # care of follow-up processing.+ self.processHost(addr, parent, False)+ else:+ self.processHost(addr, parent, True)+ s += 1++ # Resolve an IP+ def resolveIP(self, ipaddr):+ ret = list()+ self.sf.debug("Performing reverse-resolve of " + ipaddr)++ if ipaddr in self.resolveCache:+ self.sf.debug("Returning cached result for " + ipaddr + " (" ++ str(self.resolveCache[ipaddr]) + ")")+ return self.resolveCache[ipaddr]++ try:+ addrs = self.sf.normalizeDNS(socket.gethostbyaddr(ipaddr))+ self.resolveCache[ipaddr] = addrs+ self.sf.debug("Resolved " + ipaddr + " to: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to resolve " + ipaddr + " (" + str(e) + ")")+ self.resolveCache[ipaddr] = list()+ return ret++ # Resolve a host+ def resolveHost(self, hostname):+ if hostname in self.resolveCache:+ self.sf.debug("Returning cached result for " + hostname + " (" ++ str(self.resolveCache[hostname]) + ")")+ return self.resolveCache[hostname]++ try:+ addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))+ self.resolveCache[hostname] = addrs+ self.sf.debug("Resolved " + hostname + " to: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")+ return list()++ # Resolve a host to IPv6+ def resolveHost6(self, hostname):+ if hostname in self.resolveCache6:+ self.sf.debug("Returning IPv6 cached result for " + hostname + " (" ++ str(self.resolveCache6[hostname]) + ")")+ return self.resolveCache6[hostname]++ try:+ addrs = list()+ res = socket.getaddrinfo(hostname, None, socket.AF_INET6)+ for addr in res:+ if addr[4][0] not in addrs:+ addrs.append(addr[4][0])+ if len(addrs) < 1:+ return None+ self.resolveCache6[hostname] = addrs+ self.sf.debug("Resolved " + hostname + " to IPv6: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to IPv6 resolve " + hostname + " (" + str(e) + ")")+ return list()++ # Process a host/IP, parentEvent is the event that represents this entity+ def processHost(self, host, parentEvent, affiliate=None):+ parentHash = self.sf.hashstring(parentEvent.data)+ if host not in self.hostresults:+ self.hostresults[host] = list(parentHash)+ else:+ if parentHash in self.hostresults[host] or parentEvent.data == host:+ self.sf.debug("Skipping host, " + host + ", already processed.")+ return None+ else:+ self.hostresults[host].append(parentHash)++ self.sf.debug("Found host: " + host)+ # If the returned hostname is aliaseed to our+ # target in some way, flag it as an affiliate+ if affiliate is None:+ affil = True+ if self.getTarget().matches(host):+ affil = False+ # If the IP the host resolves to is in our+ # list of aliases,+ if not self.sf.validIP(host):+ for hostip in self.resolveHost(host):+ if self.getTarget().matches(hostip):+ affil = False+ else:+ affil = affiliate++ if affil:+ if self.sf.validIP(host):+ htype = "AFFILIATE_IPADDR"+ else:+ htype = "AFFILIATE_INTERNET_NAME"+ else:+ if self.sf.validIP(host):+ htype = "IP_ADDRESS"+ else:+ htype = "INTERNET_NAME"++ # Report the host+ evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)+ self.notifyListeners(evt)++ return evt++# End of sfp_dnsneighbor classdiff --git a/modules/sfp_dnsraw.py b/modules/sfp_dnsraw.pynew file mode 100644index 00000000..271260ef--- /dev/null+++ b/modules/sfp_dnsraw.py@@ -0,0 +1,121 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_dnsraw+# Purpose: SpiderFoot plug-in for collecting raw DNS records.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 07/07/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import socket+import re+import dns+import urllib2+from netaddr import IPAddress, IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_dnsraw(SpiderFootPlugin):+ """DNS Raw Records:Footprint,Investigate:DNS::Retrieves raw DNS records such as MX, TXT and others."""++ # Default options+ opts = {+ }++ # Option descriptions+ optdescs = {+ }++ events = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.events = dict()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['INTERNET_NAME', 'DOMAIN_NAME']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["PROVIDER_MAIL", "PROVIDER_DNS", "RAW_DNS_RECORDS",+ "DNS_TEXT", "DNS_SPF", "AFFILIATE_INTERNET_NAME"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data+ eventDataHash = self.sf.hashstring(eventData)+ addrs = None+ parentEvent = event++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventDataHash in self.events:+ self.sf.debug("Skipping duplicate event for " + eventData)+ return None++ self.events[eventDataHash] = True++ self.sf.debug("Gathering DNS records for " + eventData)+ # Process the raw data alone+ recdata = dict()+ recs = {+ 'MX': ['\S+\s+(?:\d+)?\s+IN\s+MX\s+\d+\s+(\S+)\.', 'PROVIDER_MAIL'],+ 'NS': ['\S+\s+(?:\d+)?\s+IN\s+NS\s+(\S+)\.', 'PROVIDER_DNS'],+ 'TXT': ['\S+\s+TXT\s+\"(.[^\"]*)"', 'DNS_TEXT']+ }++ for rec in recs.keys():+ if self.checkForStop():+ return None++ try:+ req = dns.message.make_query(eventData, dns.rdatatype.from_text(rec))++ if self.opts['_dnsserver'] != "":+ n = self.opts['_dnsserver']+ else:+ ns = dns.resolver.get_default_resolver()+ n = ns.nameservers[0]++ res = dns.query.udp(req, n, timeout=30)+ for x in res.answer:+ for rx in recs.keys():+ self.sf.debug("Checking " + str(x) + " + against " + recs[rx][0])+ pat = re.compile(recs[rx][0], re.IGNORECASE | re.DOTALL)+ grps = re.findall(pat, str(x))+ if len(grps) > 0:+ for m in grps:+ self.sf.debug("Matched: " + m)+ strdata = unicode(m, 'utf-8', errors='replace')+ evt = SpiderFootEvent(recs[rx][1], strdata,+ self.__name__, parentEvent)+ self.notifyListeners(evt)+ if rec != "TXT" and not strdata.endswith(eventData):+ evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME",+ strdata, self.__name__, parentEvent)+ self.notifyListeners(evt)+ if rec == "TXT" and "v=spf" in strdata:+ evt = SpiderFootEvent("DNS_SPF", strdata,+ self.__name__, parentEvent)+ self.notifyListeners(evt)++ else:+ strdata = unicode(str(x), 'utf-8', errors='replace')+ evt = SpiderFootEvent("RAW_DNS_RECORDS", strdata,+ self.__name__, parentEvent)+ self.notifyListeners(evt)+ except BaseException as e:+ self.sf.error("Failed to obtain DNS response for " + eventData ++ "(" + rec + "): " + str(e), False)++# End of sfp_dnsraw classdiff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.pynew file mode 100644index 00000000..9349c7cf--- /dev/null+++ b/modules/sfp_dnsresolve.py@@ -0,0 +1,345 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_dnsresolve+# Purpose: SpiderFoot plug-in for extracting hostnames from identified data+# and resolving them.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 07/07/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import socket+import re+import dns+import urllib2+from netaddr import IPAddress, IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_dnsresolve(SpiderFootPlugin):+ """DNS Resolver:Footprint,Investigate:DNS::Resolves Hosts and IP Addresses identified, also extracted from raw content."""++ # Default options+ opts = {+ 'onlyactive': True,+ 'validatereverse': True,+ 'skipcommononwildcard': True,+ 'netblocklookup': True,+ 'maxnetblock': 24+ }++ # Option descriptions+ optdescs = {+ 'skipcommononwildcard': "If wildcard DNS is detected, only attempt to look up the first common sub-domain from the common sub-domain list.",+ 'onlyactive': "Only report sub-domains/hostnames that resolve to an IP.",+ 'validatereverse': "Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.",+ 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?",+ 'maxnetblock': "Maximum owned netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"+ }++ events = dict()+ domresults = dict()+ hostresults = dict()+ resolveCache = dict()+ resolveCache6 = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.events = dict()+ self.domresults = dict()+ self.hostresults = dict()+ self.resolveCache = dict()+ self.resolveCache6 = dict()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ def enrichTarget(self, target):+ ret = list()+ # If it's an IP, get the hostname it reverse resolves to+ if target.getType() == "IP_ADDRESS":+ ret = self.resolveIP(target.getValue())+ if target.getType() == "INTERNET_NAME":+ ret = self.resolveHost(target.getValue())+ if target.getType() == "NETBLOCK_OWNER":+ ret = list()+ for addr in IPNetwork(target.getValue()):+ ipaddr = str(addr)+ if ipaddr.split(".")[3] in ['255', '0']:+ continue+ if '255' in ipaddr.split("."):+ continue+ ret.append(ipaddr)++ # Add the reverse-resolved hostnames as aliases too..+ names = self.resolveIP(ipaddr)++ if self.opts['validatereverse']:+ for host in names:+ chk = self.resolveHost(host)+ if ipaddr in chk:+ ret.append(host)+ else:+ ret.extend(names)++ for host in ret:+ if self.sf.validIP(host):+ target.setAlias(host, "IP_ADDRESS")+ else:+ target.setAlias(host, "INTERNET_NAME")+ # If the target was a hostname/sub-domain, we can+ # add the domain as an alias for the target. But+ # not if the target was an IP or subnet.+ #if target.getType() == "INTERNET_NAME":+ # dom = self.sf.hostDomain(host, self.opts['_internettlds'])+ # target.setAlias(dom, "INTERNET_NAME")++ self.sf.debug("Aliases identified: " + str(target.getAliases()))++ return target++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['*']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",+ "AFFILIATE_IPADDR", "DOMAIN_NAME", "IPV6_ADDRESS",+ "DOMAIN_NAME_PARENT"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data+ eventDataHash = self.sf.hashstring(eventData)+ addrs = None+ parentEvent = event++ # Don't be recursive+ if srcModuleName == "sfp_dnsresolve" or eventName == "ROOT":+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventDataHash in self.events:+ self.sf.debug("Skipping duplicate event for " + eventData)+ return None+ self.events[eventDataHash] = True++ # Search for IPs/hosts in raw data, but obviously nothing this module+ # already produces, as those things are already entities, not raw data.+ if eventName not in self.producedEvents():+ data = urllib2.unquote(eventData)+ for name in self.getTarget().getNames():+ if self.checkForStop():+ return None+ pat = re.compile("(%..)?([a-zA-Z0-9\-\.]+\." + name + ")", re.IGNORECASE)+ matches = re.findall(pat, data)+ if matches is not None:+ for match in matches:+ self.processHost(match[1], parentEvent, False)+ # Nothing left to do with internal links and raw data+ return None++ if eventName == 'NETBLOCK_OWNER' and self.opts['netblocklookup']:+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxnetblock']))+ return None++ # Not handling IPv6 (yet)+ if "::" in eventData:+ return None++ self.sf.debug("Looking up IPs in owned netblock: " + eventData)+ for ip in IPNetwork(eventData):+ ipaddr = str(ip)+ if "::" in ipaddr:+ continue+ if ipaddr.split(".")[3] in ['255', '0']:+ continue+ if '255' in ipaddr.split("."):+ continue++ if self.checkForStop():+ return None+ addrs = self.resolveIP(ipaddr)++ if len(addrs) > 0:+ self.sf.debug("Found a reversed hostname from " + ipaddr ++ " (" + str(addrs) + ")")+ for addr in addrs:+ # Generate an event for the IP, then+ # let the handling by this module take+ # care of follow-up processing.+ if self.checkForStop():+ return None++ self.processHost(ipaddr, parentEvent, False)+ return None++ if eventName in ["IP_ADDRESS", "INTERNET_NAME",+ "AFFILIATE_IPADDR", "AFFILIATE_INTERNET_NAME"]:++ if "_NAME" in eventName:+ addrs = self.resolveHost(eventData)+ else:+ addrs = self.resolveIP(eventData)++ # We now have a set of hosts/IPs to do something with.+ for addr in addrs:+ if self.checkForStop():+ return None++ # Does the host/IP match to the original target?+ if self.getTarget().matches(addr):+ self.processHost(addr, parentEvent, False)+ else:+ # IP addresses resolved from hosts are assumed+ # to be part of the target (non-affiliates).+ if self.sf.validIP(addr) and "AFFILIATE_" not in eventName:+ self.processHost(addr, parentEvent, False)+ else:+ self.processHost(addr, parentEvent, True)++ # Resolve an IP+ def resolveIP(self, ipaddr):+ ret = list()+ self.sf.debug("Performing reverse-resolve of " + ipaddr)++ if ipaddr in self.resolveCache:+ self.sf.debug("Returning cached result for " + ipaddr + " (" ++ str(self.resolveCache[ipaddr]) + ")")+ return self.resolveCache[ipaddr]++ try:+ addrs = self.sf.normalizeDNS(socket.gethostbyaddr(ipaddr))+ self.resolveCache[ipaddr] = addrs+ self.sf.debug("Resolved " + ipaddr + " to: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to resolve " + ipaddr + " (" + str(e) + ")")+ self.resolveCache[ipaddr] = list()+ return ret++ # Resolve a host+ def resolveHost(self, hostname):+ if hostname in self.resolveCache:+ self.sf.debug("Returning cached result for " + hostname + " (" ++ str(self.resolveCache[hostname]) + ")")+ return self.resolveCache[hostname]++ try:+ addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))+ self.resolveCache[hostname] = addrs+ self.sf.debug("Resolved " + hostname + " to: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")+ return list()++ # Resolve a host to IPv6+ def resolveHost6(self, hostname):+ if hostname in self.resolveCache6:+ self.sf.debug("Returning IPv6 cached result for " + hostname + " (" ++ str(self.resolveCache6[hostname]) + ")")+ return self.resolveCache6[hostname]++ try:+ addrs = list()+ res = socket.getaddrinfo(hostname, None, socket.AF_INET6)+ for addr in res:+ if addr[4][0] not in addrs:+ addrs.append(addr[4][0])+ if len(addrs) < 1:+ return None+ self.resolveCache6[hostname] = addrs+ self.sf.debug("Resolved " + hostname + " to IPv6: " + str(addrs))+ return addrs+ except BaseException as e:+ self.sf.debug("Unable to IPv6 resolve " + hostname + " (" + str(e) + ")")+ return list()++ # Process a host/IP, parentEvent is the event that represents this entity+ def processHost(self, host, parentEvent, affiliate=None):+ parentHash = self.sf.hashstring(parentEvent.data)+ if host not in self.hostresults:+ self.hostresults[host] = list(parentHash)+ else:+ if parentHash in self.hostresults[host] or parentEvent.data == host:+ self.sf.debug("Skipping host, " + host + ", already processed.")+ return None+ else:+ self.hostresults[host].append(parentHash)++ self.sf.debug("Found host: " + host)+ # If the returned hostname is aliaseed to our+ # target in some way, flag it as an affiliate+ if affiliate is None:+ affil = True+ if self.getTarget().matches(host):+ affil = False+ # If the IP the host resolves to is in our+ # list of aliases,+ if not self.sf.validIP(host):+ for hostip in self.resolveHost(host):+ if self.getTarget().matches(hostip):+ affil = False+ else:+ affil = affiliate++ if affil:+ if self.sf.validIP(host):+ htype = "AFFILIATE_IPADDR"+ else:+ htype = "AFFILIATE_INTERNET_NAME"+ else:+ if self.sf.validIP(host):+ htype = "IP_ADDRESS"+ else:+ htype = "INTERNET_NAME"++ if htype.endswith("INTERNET_NAME") and self.opts['onlyactive']:+ if len(self.resolveHost(host)) == 0:+ return None++ # Report the host+ evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)+ self.notifyListeners(evt)+ # Report the domain for that host+ if htype == "INTERNET_NAME":+ dom = self.sf.hostDomain(host, self.opts['_internettlds'])+ self.processDomain(dom, evt)++ # Try obtain the IPv6 address+ for ip6 in self.resolveHost6(host):+ evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt)+ self.notifyListeners(evt6)++ return evt++ def processDomain(self, domainName, parentEvent):+ if domainName not in self.domresults:+ self.domresults[domainName] = True+ else:+ self.sf.debug("Skipping domain, " + domainName + ", already processed.")+ return None++ if self.getTarget().matches(domainName):+ domevt = SpiderFootEvent("DOMAIN_NAME", domainName,+ self.__name__, parentEvent)+ self.notifyListeners(domevt)+ else:+ domevt = SpiderFootEvent("DOMAIN_NAME_PARENT", domainName,+ self.__name__, parentEvent)+ self.notifyListeners(domevt)+ return None++# End of sfp_dnsresolve classdiff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.pynew file mode 100644index 00000000..18e136ce--- /dev/null+++ b/modules/sfp_dnszonexfer.py@@ -0,0 +1,120 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_dnszonexfer+# Purpose: SpiderFoot plug-in to attempt a DNS zone transfer.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 08/07/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import socket+import re+import dns+import urllib2+from netaddr import IPAddress, IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_dnszonexfer(SpiderFootPlugin):+ """DNS Zone Transfer:Footprint,Investigate:DNS::Attempts to perform a DNS zone transfer for domains."""++ # Default options+ opts = {+ }++ # Option descriptions+ optdescs = {+ }++ events = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.events = dict()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['DOMAIN_NAME']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["RAW_DNS_RECORDS"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data+ eventDataHash = self.sf.hashstring(eventData)+ addrs = None+ parentEvent = event++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventDataHash in self.events:+ self.sf.debug("Skipping duplicate event for " + eventData)+ return None++ self.events[eventDataHash] = True++ self.sf.debug("Gathering DNS records for " + eventData)+ # Process the raw data alone+ recdata = dict()+ recs = {+ 'MX': ['\S+\s+(?:\d+)?\s+IN\s+MX\s+\d+\s+(\S+)\.', 'PROVIDER_MAIL'],+ 'NS': ['\S+\s+(?:\d+)?\s+IN\s+NS\s+(\S+)\.', 'PROVIDER_DNS'],+ 'TXT': ['\S+\s+TXT\s+\"(.[^\"]*)"', 'DNS_TEXT']+ }++ for rec in recs.keys():+ if self.checkForStop():+ return None++ try:+ req = dns.message.make_query(eventData, dns.rdatatype.from_text(rec))++ if self.opts['_dnsserver'] != "":+ n = self.opts['_dnsserver']+ else:+ ns = dns.resolver.get_default_resolver()+ n = ns.nameservers[0]++ res = dns.query.udp(req, n, timeout=30)+ for x in res.answer:+ for rx in recs.keys():+ self.sf.debug("Checking " + str(x) + " + against " + recs[rx][0])+ pat = re.compile(recs[rx][0], re.IGNORECASE | re.DOTALL)+ grps = re.findall(pat, str(x))+ if len(grps) > 0:+ for m in grps:+ self.sf.debug("Matched: " + m)+ strdata = unicode(m, 'utf-8', errors='replace')+ evt = SpiderFootEvent(recs[rx][1], strdata,+ self.__name__, parentEvent)+ self.notifyListeners(evt)+ if rec != "TXT" and not strdata.endswith(eventData):+ evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME",+ strdata, self.__name__, parentEvent)+ self.notifyListeners(evt)+ if rec == "TXT" and "v=spf" in strdata:+ evt = SpiderFootEvent("DNS_SPF", strdata,+ self.__name__, parentEvent)+ self.notifyListeners(evt)++ else:+ strdata = unicode(str(x), 'utf-8', errors='replace')+ evt = SpiderFootEvent("RAW_DNS_RECORDS", strdata,+ self.__name__, parentEvent)+ self.notifyListeners(evt)+ except BaseException as e:+ self.sf.error("Failed to obtain DNS response for " + eventData ++ "(" + rec + "): " + str(e), False)++# End of sfp_dnszonexfer class