Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Bug fixes.

Steve Micallefe807f2cb8d48
diff --git a/modules/sfp_abusech.py b/modules/sfp_abusech.py
index 4d0e7434..4809c014 100644
--- a/modules/sfp_abusech.py
+++ b/modules/sfp_abusech.py
@@ -212,8 +212,8 @@ class sfp_abusech(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.py
index 731b5854..5168cbd7 100644
--- a/modules/sfp_alienvaultiprep.py
+++ b/modules/sfp_alienvaultiprep.py
@@ -177,8 +177,8 @@ class sfp_alienvaultiprep(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_badipscom.py b/modules/sfp_badipscom.py
index 147e03f0..c56f3943 100644
--- a/modules/sfp_badipscom.py
+++ b/modules/sfp_badipscom.py
@@ -176,8 +176,8 @@ class sfp_badipscom(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_bitcash.py b/modules/sfp_bitcash.py
index 9d0257e4..ddf2e023 100644
--- a/modules/sfp_bitcash.py
+++ b/modules/sfp_bitcash.py
@@ -173,8 +173,8 @@ class sfp_bitcash(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_blocklistde.py b/modules/sfp_blocklistde.py
index 6abbbfd5..c7920f8e 100644
--- a/modules/sfp_blocklistde.py
+++ b/modules/sfp_blocklistde.py
@@ -177,8 +177,8 @@ class sfp_blocklistde(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_clearbit.py b/modules/sfp_clearbit.py
index 567dc8af..681b5888 100644
--- a/modules/sfp_clearbit.py
+++ b/modules/sfp_clearbit.py
@@ -65,7 +65,7 @@ class sfp_clearbit(SpiderFootPlugin):
useragent="SpiderFoot", headers=headers)
if res['code'] != "200":
- self.sf.error("Return code indicates no results or potential API key failure.",
+ self.sf.error("Return code indicates no results or potential API key failure or exceeded limits.",
False)
return None
diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.py
index 83c21c68..2c16ea26 100644
--- a/modules/sfp_cybercrimetracker.py
+++ b/modules/sfp_cybercrimetracker.py
@@ -176,8 +176,8 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.py
index 3ecf064f..f0d1f1ac 100644
--- a/modules/sfp_duckduckgo.py
+++ b/modules/sfp_duckduckgo.py
@@ -17,6 +17,8 @@ class sfp_duckduckgo(SpiderFootPlugin):
"""DuckDuckGo:Footprint,Investigate,Passive:Search Engines::Query DuckDuckGo's API for descriptive information about your target."""
+
+
# Default options
opts = {
"affiliatedomains": True
@@ -75,11 +77,6 @@ class sfp_duckduckgo(SpiderFootPlugin):
try:
ret = json.loads(res['content'])
except BaseException as e:
- ret = None
-
- if ret == None:
- self.sf.error("Unable to process empty response from DuckDuckGo for: " + \
- eventData, False)
return None
if ret['Heading'] == "":
diff --git a/modules/sfp_fortinet.py b/modules/sfp_fortinet.py
index 1f8e0820..6df801d5 100644
--- a/modules/sfp_fortinet.py
+++ b/modules/sfp_fortinet.py
@@ -171,8 +171,8 @@ class sfp_fortinet(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_hosting.py b/modules/sfp_hosting.py
index 750e597d..54ec0725 100644
--- a/modules/sfp_hosting.py
+++ b/modules/sfp_hosting.py
@@ -70,8 +70,12 @@ class sfp_hosting(SpiderFootPlugin):
except BaseException as e:
continue
- if IPAddress(qaddr) > IPAddress(start) and IPAddress(qaddr) < IPAddress(end):
- return [title, url]
+ try:
+ if IPAddress(qaddr) > IPAddress(start) and IPAddress(qaddr) < IPAddress(end):
+ return [title, url]
+ except BaseException as e:
+ self.sf.debug("Encountered an issue processing an IP: " + str(e))
+ continue
return None
diff --git a/modules/sfp_isc.py b/modules/sfp_isc.py
index 5879af8c..5dcdd7f3 100644
--- a/modules/sfp_isc.py
+++ b/modules/sfp_isc.py
@@ -169,8 +169,8 @@ class sfp_isc(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_malc0de.py b/modules/sfp_malc0de.py
index 6fcdd104..744d6aa9 100644
--- a/modules/sfp_malc0de.py
+++ b/modules/sfp_malc0de.py
@@ -175,8 +175,8 @@ class sfp_malc0de(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_malwaredomainlist.py b/modules/sfp_malwaredomainlist.py
index 5dfb8b14..9161fd91 100644
--- a/modules/sfp_malwaredomainlist.py
+++ b/modules/sfp_malwaredomainlist.py
@@ -191,8 +191,8 @@ class sfp_malwaredomainlist(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_malwaredomains.py b/modules/sfp_malwaredomains.py
index 6632b9b8..14fb9efe 100644
--- a/modules/sfp_malwaredomains.py
+++ b/modules/sfp_malwaredomains.py
@@ -172,8 +172,8 @@ class sfp_malwaredomains(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_mcafee.py b/modules/sfp_mcafee.py
deleted file mode 100644
index 574e7b10..00000000
--- a/modules/sfp_mcafee.py
+++ /dev/null
@@ -1,253 +0,0 @@
-# -*- coding: utf-8 -*-
-# -------------------------------------------------------------------------------
-# Name: sfp_mcafee
-# Purpose: Checks if an ASN, IP or domain is malicious.
-#
-#
-# Created: 14/12/2013
-# Copyright: (c) Steve Micallef, 2013
-# Licence: GPL
-# -------------------------------------------------------------------------------
-
-from netaddr import IPAddress, IPNetwork
-import re
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
-
-malchecks = {
- 'McAfee Site Advisor': {
- 'id': '_mcafee',
- 'type': 'query',
- 'checks': ['domain'],
- 'url': 'http://www.siteadvisor.com/sites/{0}',
- 'badregex': ['.*This link might be dangerous.*'],
- 'goodregex': []
- }
-}
-
-class sfp_mcafee(SpiderFootPlugin):
- """McAfee SiteAdvisor:Investigate,Passive:Reputation Systems:slow:Check if a host/domain is malicious according to McAfee SiteAdvisor."""
-
-
- # Default options
- opts = {
- '_mcafee': True,
- 'checkaffiliates': True,
- 'checkcohosts': True
- }
-
- # Option descriptions
- optdescs = {
- 'checkaffiliates': "Apply checks to affiliates?",
- 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?"
- }
-
- # Be sure to completely clear any class variables in setup()
- # or you run the risk of data persisting between scan runs.
-
- results = list()
-
- def setup(self, sfc, userOpts=dict()):
- self.sf = sfc
- self.results = list()
-
- # Clear / reset any other class member variables here
- # or you risk them persisting between threads.
-
- for opt in userOpts.keys():
- self.opts[opt] = userOpts[opt]
-
- # What events is this module interested in for input
- # * = be notified about all events.
- def watchedEvents(self):
- return ["INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE"]
-
- # What events this module produces
- # This is to support the end user in selecting modules based on events
- # produced.
- def producedEvents(self):
- return ["MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME",
- "MALICIOUS_COHOST"]
-
- # Check the regexps to see whether the content indicates maliciousness
- def contentMalicious(self, content, goodregex, badregex):
- # First, check for the bad indicators
- if len(badregex) > 0:
- for rx in badregex:
- if re.match(rx, content, re.IGNORECASE | re.DOTALL):
- self.sf.debug("Found to be bad against bad regex: " + rx)
- return True
-
- # Finally, check for good indicators
- if len(goodregex) > 0:
- for rx in goodregex:
- if re.match(rx, content, re.IGNORECASE | re.DOTALL):
- self.sf.debug("Found to be good againt good regex: " + rx)
- return False
-
- # If nothing was matched, reply None
- self.sf.debug("Neither good nor bad, unknown.")
- return None
-
- # Look up 'query' type sources
- def resourceQuery(self, id, target, targetType):
- self.sf.debug("Querying " + id + " for maliciousness of " + target)
- for check in malchecks.keys():
- cid = malchecks[check]['id']
- if id == cid and malchecks[check]['type'] == "query":
- url = unicode(malchecks[check]['url'])
- res = self.sf.fetchUrl(url.format(target), timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
- if res['content'] is None:
- self.sf.error("Unable to fetch " + url.format(target), False)
- return None
- if self.contentMalicious(res['content'],
- malchecks[check]['goodregex'],
- malchecks[check]['badregex']):
- return url.format(target)
-
- return None
-
- # Look up 'list' type resources
- def resourceList(self, id, target, targetType):
- targetDom = ''
- # Get the base domain if we're supplied a domain
- if targetType == "domain":
- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])
-
- for check in malchecks.keys():
- cid = malchecks[check]['id']
- if id == cid and malchecks[check]['type'] == "list":
- data = dict()
- url = malchecks[check]['url']
- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))
- if data['content'] is None:
- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
- if data['content'] is None:
- self.sf.error("Unable to fetch " + url, False)
- return None
- else:
- self.sf.cachePut("sfmal_" + cid, data['content'])
-
- # If we're looking at netblocks
- if targetType == "netblock":
- iplist = list()
- # Get the regex, replace {0} with an IP address matcher to
- # build a list of IP.
- # Cycle through each IP and check if it's in the netblock.
- if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}",
- "(\d+\.\d+\.\d+\.\d+)")
- pat = re.compile(rx, re.IGNORECASE)
- self.sf.debug("New regex for " + check + ": " + rx)
- for line in data['content'].split('\n'):
- grp = re.findall(pat, line)
- if len(grp) > 0:
- #self.sf.debug("Adding " + grp[0] + " to list.")
- iplist.append(grp[0])
- else:
- iplist = data['content'].split('\n')
-
- for ip in iplist:
- if len(ip) < 8 or ip.startswith("#"):
- continue
- ip = ip.strip()
-
- try:
- if IPAddress(ip) in IPNetwork(target):
- self.sf.debug(ip + " found within netblock/subnet " +
- target + " in " + check)
- return url
- except Exception as e:
- self.sf.debug("Error encountered parsing: " + str(e))
- continue
-
- return None
-
- # If we're looking at hostnames/domains/IPs
- if 'regex' not in malchecks[check]:
- for line in data['content'].split('\n'):
- if line == target or (targetType == "domain" and line == targetDom):
- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
- return url
- else:
- # Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
- for line in data['content'].split('\n'):
- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
- re.match(rxTgt, line, re.IGNORECASE):
- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
- return url
- return None
-
- def lookupItem(self, resourceId, itemType, target):
- for check in malchecks.keys():
- cid = malchecks[check]['id']
- if cid == resourceId and itemType in malchecks[check]['checks']:
- self.sf.debug("Checking maliciousness of " + target + " (" +
- itemType + ") with: " + cid)
- if malchecks[check]['type'] == "query":
- return self.resourceQuery(cid, target, itemType)
- if malchecks[check]['type'] == "list":
- return self.resourceList(cid, target, itemType)
-
- return None
-
- # Handle events sent to this module
- def handleEvent(self, event):
- eventName = event.eventType
- srcModuleName = event.module
- eventData = event.data
-
- self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
-
- if eventData in self.results:
- self.sf.debug("Skipping " + eventData + ", already checked.")
- return None
- else:
- self.results.append(eventData)
-
- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):
- return None
- if eventName == 'AFFILIATE_IPADDR' \
- and not self.opts.get('checkaffiliates', False):
- return None
-
- for check in malchecks.keys():
- cid = malchecks[check]['id']
- # If the module is enabled..
- if self.opts[cid]:
- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:
- typeId = 'ip'
- if eventName == 'IP_ADDRESS':
- evtType = 'MALICIOUS_IPADDR'
- else:
- evtType = 'MALICIOUS_AFFILIATE_IPADDR'
-
- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:
- typeId = 'asn'
- evtType = 'MALICIOUS_ASN'
-
- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',
- 'AFFILIATE_INTERNET_NAME', ]:
- typeId = 'domain'
- if eventName == "INTERNET_NAME":
- evtType = "MALICIOUS_INTERNET_NAME"
- if eventName == 'AFFILIATE_INTERNET_NAME':
- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'
- if eventName == 'CO_HOSTED_SITE':
- evtType = 'MALICIOUS_COHOST'
-
- url = self.lookupItem(cid, typeId, eventData)
- if self.checkForStop():
- return None
-
- # Notify other modules of what you've found
- if url is not None:
- text = check + " [" + eventData + "]\n" + "<SFURL>" + url + "</SFURL>"
- evt = SpiderFootEvent(evtType, text, self.__name__, event)
- self.notifyListeners(evt)
-
- return None
-
-# End of sfp_mcafee class
diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.py
index b38a065f..0dfa95f0 100644
--- a/modules/sfp_multiproxy.py
+++ b/modules/sfp_multiproxy.py
@@ -171,8 +171,8 @@ class sfp_multiproxy(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_nothink.py b/modules/sfp_nothink.py
index 4e9f644b..4efce179 100644
--- a/modules/sfp_nothink.py
+++ b/modules/sfp_nothink.py
@@ -198,8 +198,8 @@ class sfp_nothink(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_phishtank.py b/modules/sfp_phishtank.py
index 123daa81..1d5efd9b 100644
--- a/modules/sfp_phishtank.py
+++ b/modules/sfp_phishtank.py
@@ -173,8 +173,8 @@ class sfp_phishtank(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_sslcert.py b/modules/sfp_sslcert.py
index c61218bf..b0da3ee2 100644
--- a/modules/sfp_sslcert.py
+++ b/modules/sfp_sslcert.py
@@ -11,7 +11,6 @@
# -------------------------------------------------------------------------------
import socket
-import socks
import ssl
import time
import M2Crypto
@@ -118,7 +117,12 @@ class sfp_sslcert(SpiderFootPlugin):
# Report back who the certificate was issued to
def getIssued(self, cert, sevt):
- issued = cert.get_subject().as_text().encode('raw_unicode_escape')
+ try:
+ issued = cert.get_subject().as_text().encode('raw_unicode_escape')
+ except BaseException as e:
+ self.sf.error("Error processing certificate: " + str(e), False)
+ return None
+
evt = SpiderFootEvent("SSL_CERTIFICATE_ISSUED", issued, self.__name__, sevt)
self.notifyListeners(evt)
diff --git a/modules/sfp_threatexpert.py b/modules/sfp_threatexpert.py
index 6dd1bf9c..e58e5591 100644
--- a/modules/sfp_threatexpert.py
+++ b/modules/sfp_threatexpert.py
@@ -179,8 +179,8 @@ class sfp_threatexpert(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.py
index 5d3401a7..c33ad6a9 100644
--- a/modules/sfp_torexits.py
+++ b/modules/sfp_torexits.py
@@ -177,8 +177,8 @@ class sfp_torexits(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_torserver.py b/modules/sfp_torserver.py
index 3c72552c..bee5aaa9 100644
--- a/modules/sfp_torserver.py
+++ b/modules/sfp_torserver.py
@@ -176,8 +176,8 @@ class sfp_torserver(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_totalhash.py b/modules/sfp_totalhash.py
index 454b4223..0c9fb338 100644
--- a/modules/sfp_totalhash.py
+++ b/modules/sfp_totalhash.py
@@ -175,8 +175,8 @@ class sfp_totalhash(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.py
index 4cb7cda0..35ee55fe 100644
--- a/modules/sfp_voipbl.py
+++ b/modules/sfp_voipbl.py
@@ -181,8 +181,8 @@ class sfp_voipbl(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.py
index ab977027..a119d904 100644
--- a/modules/sfp_vxvault.py
+++ b/modules/sfp_vxvault.py
@@ -176,8 +176,8 @@ class sfp_vxvault(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/modules/sfp_watchguard.py b/modules/sfp_watchguard.py
index d629706f..2b4d29dd 100644
--- a/modules/sfp_watchguard.py
+++ b/modules/sfp_watchguard.py
@@ -174,8 +174,8 @@ class sfp_watchguard(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom)
- rxTgt = unicode(malchecks[check]['regex']).format(target)
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
for line in data['content'].split('\n'):
if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
re.match(rxTgt, line, re.IGNORECASE):
diff --git a/sflib.py b/sflib.py
index 3507a58c..9b6b028e 100644
--- a/sflib.py
+++ b/sflib.py
@@ -971,7 +971,8 @@ class SpiderFoot:
return None
# Clean the URL
- url = url.encode('ascii', 'ignore')
+ if type(url) != unicode:
+ url = unicode(url, 'utf-8', errors='replace')
try:
header = dict()