SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Bug fixes.
Steve Micallefe807f2cb8d48
diff --git a/modules/sfp_abusech.py b/modules/sfp_abusech.pyindex 4d0e7434..4809c014 100644--- a/modules/sfp_abusech.py+++ b/modules/sfp_abusech.py@@ -212,8 +212,8 @@ class sfp_abusech(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.pyindex 731b5854..5168cbd7 100644--- a/modules/sfp_alienvaultiprep.py+++ b/modules/sfp_alienvaultiprep.py@@ -177,8 +177,8 @@ class sfp_alienvaultiprep(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_badipscom.py b/modules/sfp_badipscom.pyindex 147e03f0..c56f3943 100644--- a/modules/sfp_badipscom.py+++ b/modules/sfp_badipscom.py@@ -176,8 +176,8 @@ class sfp_badipscom(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_bitcash.py b/modules/sfp_bitcash.pyindex 9d0257e4..ddf2e023 100644--- a/modules/sfp_bitcash.py+++ b/modules/sfp_bitcash.py@@ -173,8 +173,8 @@ class sfp_bitcash(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_blocklistde.py b/modules/sfp_blocklistde.pyindex 6abbbfd5..c7920f8e 100644--- a/modules/sfp_blocklistde.py+++ b/modules/sfp_blocklistde.py@@ -177,8 +177,8 @@ class sfp_blocklistde(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_clearbit.py b/modules/sfp_clearbit.pyindex 567dc8af..681b5888 100644--- a/modules/sfp_clearbit.py+++ b/modules/sfp_clearbit.py@@ -65,7 +65,7 @@ class sfp_clearbit(SpiderFootPlugin):useragent="SpiderFoot", headers=headers)if res['code'] != "200":- self.sf.error("Return code indicates no results or potential API key failure.",+ self.sf.error("Return code indicates no results or potential API key failure or exceeded limits.",False)return Nonediff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.pyindex 83c21c68..2c16ea26 100644--- a/modules/sfp_cybercrimetracker.py+++ b/modules/sfp_cybercrimetracker.py@@ -176,8 +176,8 @@ class sfp_cybercrimetracker(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.pyindex 3ecf064f..f0d1f1ac 100644--- a/modules/sfp_duckduckgo.py+++ b/modules/sfp_duckduckgo.py@@ -17,6 +17,8 @@ class sfp_duckduckgo(SpiderFootPlugin):"""DuckDuckGo:Footprint,Investigate,Passive:Search Engines::Query DuckDuckGo's API for descriptive information about your target."""++# Default optionsopts = {"affiliatedomains": True@@ -75,11 +77,6 @@ class sfp_duckduckgo(SpiderFootPlugin):try:ret = json.loads(res['content'])except BaseException as e:- ret = None-- if ret == None:- self.sf.error("Unable to process empty response from DuckDuckGo for: " + \- eventData, False)return Noneif ret['Heading'] == "":diff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.pyindex 1f8e0820..6df801d5 100644--- a/modules/sfp_fortinet.py+++ b/modules/sfp_fortinet.py@@ -171,8 +171,8 @@ class sfp_fortinet(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_hosting.py b/modules/sfp_hosting.pyindex 750e597d..54ec0725 100644--- a/modules/sfp_hosting.py+++ b/modules/sfp_hosting.py@@ -70,8 +70,12 @@ class sfp_hosting(SpiderFootPlugin):except BaseException as e:continue- if IPAddress(qaddr) > IPAddress(start) and IPAddress(qaddr) < IPAddress(end):- return [title, url]+ try:+ if IPAddress(qaddr) > IPAddress(start) and IPAddress(qaddr) < IPAddress(end):+ return [title, url]+ except BaseException as e:+ self.sf.debug("Encountered an issue processing an IP: " + str(e))+ continuereturn Nonediff --git a/modules/sfp_isc.py b/modules/sfp_isc.pyindex 5879af8c..5dcdd7f3 100644--- a/modules/sfp_isc.py+++ b/modules/sfp_isc.py@@ -169,8 +169,8 @@ class sfp_isc(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_malc0de.py b/modules/sfp_malc0de.pyindex 6fcdd104..744d6aa9 100644--- a/modules/sfp_malc0de.py+++ b/modules/sfp_malc0de.py@@ -175,8 +175,8 @@ class sfp_malc0de(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_malwaredomainlist.py b/modules/sfp_malwaredomainlist.pyindex 5dfb8b14..9161fd91 100644--- a/modules/sfp_malwaredomainlist.py+++ b/modules/sfp_malwaredomainlist.py@@ -191,8 +191,8 @@ class sfp_malwaredomainlist(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_malwaredomains.py b/modules/sfp_malwaredomains.pyindex 6632b9b8..14fb9efe 100644--- a/modules/sfp_malwaredomains.py+++ b/modules/sfp_malwaredomains.py@@ -172,8 +172,8 @@ class sfp_malwaredomains(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_mcafee.py b/modules/sfp_mcafee.pydeleted file mode 100644index 574e7b10..00000000--- a/modules/sfp_mcafee.py+++ /dev/null@@ -1,253 +0,0 @@-# -*- coding: utf-8 -*--# --------------------------------------------------------------------------------# Name: sfp_mcafee-# Purpose: Checks if an ASN, IP or domain is malicious.-#-# Author: [email protected]-#-# Created: 14/12/2013-# Copyright: (c) Steve Micallef, 2013-# Licence: GPL-# ---------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork-import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent--malchecks = {- 'McAfee Site Advisor': {- 'id': '_mcafee',- 'type': 'query',- 'checks': ['domain'],- 'url': 'http://www.siteadvisor.com/sites/{0}',- 'badregex': ['.*This link might be dangerous.*'],- 'goodregex': []- }-}--class sfp_mcafee(SpiderFootPlugin):- """McAfee SiteAdvisor:Investigate,Passive:Reputation Systems:slow:Check if a host/domain is malicious according to McAfee SiteAdvisor."""--- # Default options- opts = {- '_mcafee': True,- 'checkaffiliates': True,- 'checkcohosts': True- }-- # Option descriptions- optdescs = {- 'checkaffiliates': "Apply checks to affiliates?",- 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?"- }-- # Be sure to completely clear any class variables in setup()- # or you run the risk of data persisting between scan runs.-- results = list()-- def setup(self, sfc, userOpts=dict()):- self.sf = sfc- self.results = list()-- # Clear / reset any other class member variables here- # or you risk them persisting between threads.-- for opt in userOpts.keys():- self.opts[opt] = userOpts[opt]-- # What events is this module interested in for input- # * = be notified about all events.- def watchedEvents(self):- return ["INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE"]-- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.- def producedEvents(self):- return ["MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME",- "MALICIOUS_COHOST"]-- # Check the regexps to see whether the content indicates maliciousness- def contentMalicious(self, content, goodregex, badregex):- # First, check for the bad indicators- if len(badregex) > 0:- for rx in badregex:- if re.match(rx, content, re.IGNORECASE | re.DOTALL):- self.sf.debug("Found to be bad against bad regex: " + rx)- return True-- # Finally, check for good indicators- if len(goodregex) > 0:- for rx in goodregex:- if re.match(rx, content, re.IGNORECASE | re.DOTALL):- self.sf.debug("Found to be good againt good regex: " + rx)- return False-- # If nothing was matched, reply None- self.sf.debug("Neither good nor bad, unknown.")- return None-- # Look up 'query' type sources- def resourceQuery(self, id, target, targetType):- self.sf.debug("Querying " + id + " for maliciousness of " + target)- for check in malchecks.keys():- cid = malchecks[check]['id']- if id == cid and malchecks[check]['type'] == "query":- url = unicode(malchecks[check]['url'])- res = self.sf.fetchUrl(url.format(target), timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if res['content'] is None:- self.sf.error("Unable to fetch " + url.format(target), False)- return None- if self.contentMalicious(res['content'],- malchecks[check]['goodregex'],- malchecks[check]['badregex']):- return url.format(target)-- return None-- # Look up 'list' type resources- def resourceList(self, id, target, targetType):- targetDom = ''- # Get the base domain if we're supplied a domain- if targetType == "domain":- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])-- for check in malchecks.keys():- cid = malchecks[check]['id']- if id == cid and malchecks[check]['type'] == "list":- data = dict()- url = malchecks[check]['url']- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))- if data['content'] is None:- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if data['content'] is None:- self.sf.error("Unable to fetch " + url, False)- return None- else:- self.sf.cachePut("sfmal_" + cid, data['content'])-- # If we're looking at netblocks- if targetType == "netblock":- iplist = list()- # Get the regex, replace {0} with an IP address matcher to- # build a list of IP.- # Cycle through each IP and check if it's in the netblock.- if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")- pat = re.compile(rx, re.IGNORECASE)- self.sf.debug("New regex for " + check + ": " + rx)- for line in data['content'].split('\n'):- grp = re.findall(pat, line)- if len(grp) > 0:- #self.sf.debug("Adding " + grp[0] + " to list.")- iplist.append(grp[0])- else:- iplist = data['content'].split('\n')-- for ip in iplist:- if len(ip) < 8 or ip.startswith("#"):- continue- ip = ip.strip()-- try:- if IPAddress(ip) in IPNetwork(target):- self.sf.debug(ip + " found within netblock/subnet " +- target + " in " + check)- return url- except Exception as e:- self.sf.debug("Error encountered parsing: " + str(e))- continue-- return None-- # If we're looking at hostnames/domains/IPs- if 'regex' not in malchecks[check]:- for line in data['content'].split('\n'):- if line == target or (targetType == "domain" and line == targetDom):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- else:- # Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- return None-- def lookupItem(self, resourceId, itemType, target):- for check in malchecks.keys():- cid = malchecks[check]['id']- if cid == resourceId and itemType in malchecks[check]['checks']:- self.sf.debug("Checking maliciousness of " + target + " (" +- itemType + ") with: " + cid)- if malchecks[check]['type'] == "query":- return self.resourceQuery(cid, target, itemType)- if malchecks[check]['type'] == "list":- return self.resourceList(cid, target, itemType)-- return None-- # Handle events sent to this module- def handleEvent(self, event):- eventName = event.eventType- srcModuleName = event.module- eventData = event.data-- self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)-- if eventData in self.results:- self.sf.debug("Skipping " + eventData + ", already checked.")- return None- else:- self.results.append(eventData)-- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):- return None- if eventName == 'AFFILIATE_IPADDR' \- and not self.opts.get('checkaffiliates', False):- return None-- for check in malchecks.keys():- cid = malchecks[check]['id']- # If the module is enabled..- if self.opts[cid]:- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:- typeId = 'ip'- if eventName == 'IP_ADDRESS':- evtType = 'MALICIOUS_IPADDR'- else:- evtType = 'MALICIOUS_AFFILIATE_IPADDR'-- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:- typeId = 'asn'- evtType = 'MALICIOUS_ASN'-- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',- 'AFFILIATE_INTERNET_NAME', ]:- typeId = 'domain'- if eventName == "INTERNET_NAME":- evtType = "MALICIOUS_INTERNET_NAME"- if eventName == 'AFFILIATE_INTERNET_NAME':- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'- if eventName == 'CO_HOSTED_SITE':- evtType = 'MALICIOUS_COHOST'-- url = self.lookupItem(cid, typeId, eventData)- if self.checkForStop():- return None-- # Notify other modules of what you've found- if url is not None:- text = check + " [" + eventData + "]\n" + "<SFURL>" + url + "</SFURL>"- evt = SpiderFootEvent(evtType, text, self.__name__, event)- self.notifyListeners(evt)-- return None--# End of sfp_mcafee classdiff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.pyindex b38a065f..0dfa95f0 100644--- a/modules/sfp_multiproxy.py+++ b/modules/sfp_multiproxy.py@@ -171,8 +171,8 @@ class sfp_multiproxy(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_nothink.py b/modules/sfp_nothink.pyindex 4e9f644b..4efce179 100644--- a/modules/sfp_nothink.py+++ b/modules/sfp_nothink.py@@ -198,8 +198,8 @@ class sfp_nothink(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_phishtank.py b/modules/sfp_phishtank.pyindex 123daa81..1d5efd9b 100644--- a/modules/sfp_phishtank.py+++ b/modules/sfp_phishtank.py@@ -173,8 +173,8 @@ class sfp_phishtank(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_sslcert.py b/modules/sfp_sslcert.pyindex c61218bf..b0da3ee2 100644--- a/modules/sfp_sslcert.py+++ b/modules/sfp_sslcert.py@@ -11,7 +11,6 @@# -------------------------------------------------------------------------------import socket-import socksimport sslimport timeimport M2Crypto@@ -118,7 +117,12 @@ class sfp_sslcert(SpiderFootPlugin):# Report back who the certificate was issued todef getIssued(self, cert, sevt):- issued = cert.get_subject().as_text().encode('raw_unicode_escape')+ try:+ issued = cert.get_subject().as_text().encode('raw_unicode_escape')+ except BaseException as e:+ self.sf.error("Error processing certificate: " + str(e), False)+ return None+evt = SpiderFootEvent("SSL_CERTIFICATE_ISSUED", issued, self.__name__, sevt)self.notifyListeners(evt)diff --git a/modules/sfp_threatexpert.py b/modules/sfp_threatexpert.pyindex 6dd1bf9c..e58e5591 100644--- a/modules/sfp_threatexpert.py+++ b/modules/sfp_threatexpert.py@@ -179,8 +179,8 @@ class sfp_threatexpert(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.pyindex 5d3401a7..c33ad6a9 100644--- a/modules/sfp_torexits.py+++ b/modules/sfp_torexits.py@@ -177,8 +177,8 @@ class sfp_torexits(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_torserver.py b/modules/sfp_torserver.pyindex 3c72552c..bee5aaa9 100644--- a/modules/sfp_torserver.py+++ b/modules/sfp_torserver.py@@ -176,8 +176,8 @@ class sfp_torserver(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_totalhash.py b/modules/sfp_totalhash.pyindex 454b4223..0c9fb338 100644--- a/modules/sfp_totalhash.py+++ b/modules/sfp_totalhash.py@@ -175,8 +175,8 @@ class sfp_totalhash(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.pyindex 4cb7cda0..35ee55fe 100644--- a/modules/sfp_voipbl.py+++ b/modules/sfp_voipbl.py@@ -181,8 +181,8 @@ class sfp_voipbl(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.pyindex ab977027..a119d904 100644--- a/modules/sfp_vxvault.py+++ b/modules/sfp_vxvault.py@@ -176,8 +176,8 @@ class sfp_vxvault(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/modules/sfp_watchguard.py b/modules/sfp_watchguard.pyindex d629706f..2b4d29dd 100644--- a/modules/sfp_watchguard.py+++ b/modules/sfp_watchguard.py@@ -174,8 +174,8 @@ class sfp_watchguard(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom)- rxTgt = unicode(malchecks[check]['regex']).format(target)+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))for line in data['content'].split('\n'):if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \re.match(rxTgt, line, re.IGNORECASE):diff --git a/sflib.py b/sflib.pyindex 3507a58c..9b6b028e 100644--- a/sflib.py+++ b/sflib.py@@ -971,7 +971,8 @@ class SpiderFoot:return None# Clean the URL- url = url.encode('ascii', 'ignore')+ if type(url) != unicode:+ url = unicode(url, 'utf-8', errors='replace')try:header = dict()