Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Consistent treatment of query params.

smicallef54849d324ede
diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.py
index 1b5f1329..f5ba90af 100644
--- a/modules/sfp_citadel.py
+++ b/modules/sfp_citadel.py
@@ -57,7 +57,7 @@ class sfp_citadel(SpiderFootPlugin):
apikey = "3edfb5603418f101926c64ca5dd0e409"
params = {
- 'query': email.encode('raw_unicode_escape').decode("ascii"),
+ 'query': email.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'type': 'email_address',
'key': apikey
}
diff --git a/modules/sfp_darksearch.py b/modules/sfp_darksearch.py
index a27ed779..72d634ba 100644
--- a/modules/sfp_darksearch.py
+++ b/modules/sfp_darksearch.py
@@ -52,7 +52,7 @@ class sfp_darksearch(SpiderFootPlugin):
# https://darksearch.io/apidoc
def query(self, qry, page):
params = {
- 'query': '"' + qry.encode('raw_unicode_escape').decode("ascii") + '"',
+ 'query': '"' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace') + '"',
'page': str(page)
}
diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.py
index 40824c3f..51a7e8ad 100644
--- a/modules/sfp_fringeproject.py
+++ b/modules/sfp_fringeproject.py
@@ -45,7 +45,7 @@ class sfp_fringeproject(SpiderFootPlugin):
def query(self, qry):
params = {
- 'q': qry.encode('raw_unicode_escape').decode("ascii")
+ 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
res = self.sf.fetchUrl('https://api.fringeproject.com/api/search?' + urllib.parse.urlencode(params),
diff --git a/modules/sfp_fsecure_riddler.py b/modules/sfp_fsecure_riddler.py
index 82e83a4b..a6b1e944 100644
--- a/modules/sfp_fsecure_riddler.py
+++ b/modules/sfp_fsecure_riddler.py
@@ -95,7 +95,7 @@ class sfp_fsecure_riddler(SpiderFootPlugin):
# https://riddler.io/help/search
def query(self, qry):
params = {
- 'query': qry.encode('raw_unicode_escape').decode("ascii")
+ 'query': qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
headers = {
'Authentication-Token': self.token,
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index 2e4e909f..df49de39 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -53,7 +53,7 @@ class sfp_gravatar(SpiderFootPlugin):
# https://secure.gravatar.com/site/implement/
# https://secure.gravatar.com/site/implement/profiles/
def query(self, qry):
- email_hash = hashlib.md5(qry.encode('utf-8').lower()).hexdigest()
+ email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest()
output = 'json'
res = self.sf.fetchUrl("https://secure.gravatar.com/" + email_hash + '.' + output,
diff --git a/modules/sfp_numverify.py b/modules/sfp_numverify.py
index f8b906ff..6f76c768 100644
--- a/modules/sfp_numverify.py
+++ b/modules/sfp_numverify.py
@@ -54,7 +54,7 @@ class sfp_numverify(SpiderFootPlugin):
number = qry.strip('+').strip('(').strip(')')
params = {
- 'number': number.encode('raw_unicode_escape').decode("ascii"),
+ 'number': number.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'country_code': '',
'format': '0', # set to "1" for prettified debug output
'access_key': self.opts['api_key']
diff --git a/modules/sfp_onionsearchengine.py b/modules/sfp_onionsearchengine.py
index 7baf801e..88131fb5 100644
--- a/modules/sfp_onionsearchengine.py
+++ b/modules/sfp_onionsearchengine.py
@@ -73,7 +73,7 @@ class sfp_onionsearchengine(SpiderFootPlugin):
return None
params = {
- 'search': '"' + eventData.encode('raw_unicode_escape').decode("ascii") + '"',
+ 'search': '"' + eventData.encode('raw_unicode_escape').decode("ascii", errors='replace') + '"',
'submit': 'Search',
'page': str(page)
}
diff --git a/modules/sfp_pulsedive.py b/modules/sfp_pulsedive.py
index d2099cb4..773c7ac3 100644
--- a/modules/sfp_pulsedive.py
+++ b/modules/sfp_pulsedive.py
@@ -73,7 +73,7 @@ class sfp_pulsedive(SpiderFootPlugin):
# https://pulsedive.com/api/
def query(self, qry):
params = {
- 'indicator': qry.encode('raw_unicode_escape').decode("ascii"),
+ 'indicator': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'key': self.opts['api_key']
}
diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.py
index a7d18006..2da3b2e7 100644
--- a/modules/sfp_scylla.py
+++ b/modules/sfp_scylla.py
@@ -53,7 +53,7 @@ class sfp_scylla(SpiderFootPlugin):
# Query Scylla API
def query(self, qry, per_page=20, start=0):
params = {
- 'q': 'Email:@' + qry.encode('raw_unicode_escape').decode("ascii"),
+ 'q': 'Email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'num': str(per_page),
'from': str(start)
}
diff --git a/modules/sfp_shodan.py b/modules/sfp_shodan.py
index 3711a946..f9700a0c 100644
--- a/modules/sfp_shodan.py
+++ b/modules/sfp_shodan.py
@@ -92,7 +92,7 @@ class sfp_shodan(SpiderFootPlugin):
def searchHtml(self, qry):
params = {
- 'query': 'http.html:"' + qry.encode('raw_unicode_escape').decode("ascii") + '"',
+ 'query': 'http.html:"' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace') + '"',
'key': self.opts['api_key']
}
diff --git a/modules/sfp_ssltools.py b/modules/sfp_ssltools.py
index 43fda691..8a09231e 100644
--- a/modules/sfp_ssltools.py
+++ b/modules/sfp_ssltools.py
@@ -58,7 +58,7 @@ class sfp_ssltools(SpiderFootPlugin):
# Query SSL Tools for DNS
def queryDns(self, domain):
params = {
- 'url': domain.encode('raw_unicode_escape').decode("ascii")
+ 'url': domain.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
headers = { 'Content-Type': 'application/x-www-form-urlencoded' }
@@ -86,7 +86,7 @@ class sfp_ssltools(SpiderFootPlugin):
# Query SSL Tools for certificate information
def queryScan(self, domain, port):
params = {
- 'url': domain.encode('raw_unicode_escape').decode("ascii"),
+ 'url': domain.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'path': '/',
'port': str(port),
'live_scan': 'true'
diff --git a/modules/sfp_urlscan.py b/modules/sfp_urlscan.py
index dea0ee6e..8ceed67c 100644
--- a/modules/sfp_urlscan.py
+++ b/modules/sfp_urlscan.py
@@ -48,7 +48,7 @@ class sfp_urlscan(SpiderFootPlugin):
# https://urlscan.io/about-api/
def query(self, qry):
params = {
- 'q': 'domain:' + qry.encode('raw_unicode_escape').decode("ascii")
+ 'q': 'domain:' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace')
}
res = self.sf.fetchUrl('https://urlscan.io/api/v1/search/?' + urllib.parse.urlencode(params),
diff --git a/modules/sfp_whatcms.py b/modules/sfp_whatcms.py
index 7505370f..64755859 100644
--- a/modules/sfp_whatcms.py
+++ b/modules/sfp_whatcms.py
@@ -62,7 +62,7 @@ class sfp_whatcms(SpiderFootPlugin):
# https://whatcms.org/Documentation
def queryCmsDetect(self, qry):
params = {
- 'url': qry,
+ 'url': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'key': self.opts['api_key']
}
@@ -78,7 +78,7 @@ class sfp_whatcms(SpiderFootPlugin):
# https://whatcms.org/Documentation
def queryCmsTechnology(self, qry):
params = {
- 'url': qry,
+ 'url': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
'key': self.opts['api_key']
}