Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

modules: cast fetchUrl content as string in calls to findall() (#1339)

bcoles8213ee5c57d3
diff --git a/modules/sfp_callername.py b/modules/sfp_callername.py
index 9aa7b85f..1e290faf 100644
--- a/modules/sfp_callername.py
+++ b/modules/sfp_callername.py
@@ -110,7 +110,7 @@ class sfp_callername(SpiderFootPlugin):
self.sf.debug('No phone information found for ' + eventData)
return
- location_match = re.findall(r'<div class="callerid"><h4>.*?</h4><p>(.+?)</p></div>', res['content'], re.MULTILINE | re.DOTALL)
+ location_match = re.findall(r'<div class="callerid"><h4>.*?</h4><p>(.+?)</p></div>', str(res['content']), re.MULTILINE | re.DOTALL)
if location_match:
location = location_match[0]
@@ -121,8 +121,8 @@ class sfp_callername(SpiderFootPlugin):
evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)
self.notifyListeners(evt)
- rep_good_match = re.findall(r'>SAFE.*?>(\d+) votes?<', res['content'])
- rep_bad_match = re.findall(r'>UNSAFE.*?>(\d+) votes?<', res['content'])
+ rep_good_match = re.findall(r'>SAFE.*?>(\d+) votes?<', str(res['content']))
+ rep_bad_match = re.findall(r'>UNSAFE.*?>(\d+) votes?<', str(res['content']))
if rep_good_match and rep_bad_match:
good_votes = int(rep_good_match[0])
diff --git a/modules/sfp_commoncrawl.py b/modules/sfp_commoncrawl.py
index 74ce3e44..394a74d1 100644
--- a/modules/sfp_commoncrawl.py
+++ b/modules/sfp_commoncrawl.py
@@ -101,7 +101,7 @@ class sfp_commoncrawl(SpiderFootPlugin):
self.errorState = True
return list()
- indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", res['content'])
+ indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", str(res['content']))
indexlist = dict()
for m in indexes:
ms = m.replace("CC-MAIN-", "").replace("-", "")
diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.py
index 430d838b..ed9798fb 100644
--- a/modules/sfp_crossref.py
+++ b/modules/sfp_crossref.py
@@ -113,7 +113,7 @@ class sfp_crossref(SpiderFootPlugin):
r"([\.\'\/\"\ ]" + re.escape(name) + r"[\.\'\/\"\ ])",
re.IGNORECASE
)
- matches = re.findall(pat, res['content'])
+ matches = re.findall(pat, str(res['content']))
if len(matches) > 0:
matched = True
@@ -144,7 +144,7 @@ class sfp_crossref(SpiderFootPlugin):
r"([\.\'\/\"\ ]" + re.escape(name) + r"[\'\/\"\ ])",
re.IGNORECASE
)
- matches = re.findall(pat, res['content'])
+ matches = re.findall(pat, str(res['content']))
if len(matches) > 0:
matched = True
diff --git a/modules/sfp_flickr.py b/modules/sfp_flickr.py
index 9a7318eb..8c78578f 100644
--- a/modules/sfp_flickr.py
+++ b/modules/sfp_flickr.py
@@ -91,7 +91,7 @@ class sfp_flickr(SpiderFootPlugin):
if res['content'] is None:
return None
- keys = re.findall(r'YUI_config.flickr.api.site_key = "([a-zA-Z0-9]+)"', res['content'])
+ keys = re.findall(r'YUI_config.flickr.api.site_key = "([a-zA-Z0-9]+)"', str(res['content']))
if not keys:
return None
diff --git a/modules/sfp_h1nobbdde.py b/modules/sfp_h1nobbdde.py
index 89b0fdff..9cca1c40 100644
--- a/modules/sfp_h1nobbdde.py
+++ b/modules/sfp_h1nobbdde.py
@@ -77,7 +77,7 @@ class sfp_h1nobbdde(SpiderFootPlugin):
try:
rx = re.compile("<a class=\"title\" href=.(.[^\"]+).*?title=.(.[^\"\']+)", re.IGNORECASE | re.DOTALL)
- for m in rx.findall(res['content']):
+ for m in rx.findall(str(res['content'])):
# Report it
if qry in m[1]:
ret.append(m[1] + "\n<SFURL>" + m[0] + "</SFURL>")
diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.py
index e96a8072..e74390e8 100644
--- a/modules/sfp_hackertarget.py
+++ b/modules/sfp_hackertarget.py
@@ -130,7 +130,7 @@ class sfp_hackertarget(SpiderFootPlugin):
if res['content'] is None:
return None
- html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', res['content'], re.MULTILINE | re.DOTALL)
+ html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', str(res['content']), re.MULTILINE | re.DOTALL)
if not html_data:
self.sf.debug(f"Found no open UDP ports on {ip}")
diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.py
index 791bd315..9cbb57e2 100644
--- a/modules/sfp_myspace.py
+++ b/modules/sfp_myspace.py
@@ -84,7 +84,7 @@ class sfp_myspace(SpiderFootPlugin):
return
# Extract HTML containing potential profile matches
- profiles = re.findall(r'<a href="/[a-zA-Z0-9_]+">[^<]+</a></h6>', res['content'])
+ profiles = re.findall(r'<a href="/[a-zA-Z0-9_]+">[^<]+</a></h6>', str(res['content']))
if not profiles:
self.sf.debug(f"No profiles found for e-mail: {email}")
diff --git a/modules/sfp_open_passive_dns_database.py b/modules/sfp_open_passive_dns_database.py
index 4cc9b6a2..0d784911 100644
--- a/modules/sfp_open_passive_dns_database.py
+++ b/modules/sfp_open_passive_dns_database.py
@@ -93,7 +93,7 @@ class sfp_open_passive_dns_database(SpiderFootPlugin):
self.sf.debug("No results found for " + qry)
return None
- rows = re.findall(r'<tr>(.+?)</tr>', res['content'], re.DOTALL)
+ rows = re.findall(r'<tr>(.+?)</tr>', str(res['content']), re.DOTALL)
if not rows:
self.sf.debug("No passive DNS results for " + qry)
diff --git a/modules/sfp_openbugbounty.py b/modules/sfp_openbugbounty.py
index de73c2b2..17a0a609 100644
--- a/modules/sfp_openbugbounty.py
+++ b/modules/sfp_openbugbounty.py
@@ -84,7 +84,7 @@ class sfp_openbugbounty(SpiderFootPlugin):
try:
rx = re.compile(".*<div class=.cell1.><a href=.(.*).>(.*" + qry + ").*?</a></div>.*", re.IGNORECASE)
- for m in rx.findall(res['content']):
+ for m in rx.findall(str(res['content'])):
# Report it
if m[1] == qry or m[1].endswith("." + qry):
ret.append("From openbugbounty.org: <SFURL>" + base + m[0] + "</SFURL>")
diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.py
index 6ecf5760..af4f3cb5 100644
--- a/modules/sfp_pgp.py
+++ b/modules/sfp_pgp.py
@@ -115,7 +115,7 @@ class sfp_pgp(SpiderFootPlugin):
if res['content'] is not None and res['code'] != "503":
pat = re.compile("(-----BEGIN.*END.*BLOCK-----)", re.MULTILINE | re.DOTALL)
- matches = re.findall(pat, res['content'])
+ matches = re.findall(pat, str(res['content']))
for match in matches:
self.sf.debug("Found public key: " + match)
if len(match) < 300:
diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.py
index d5eac401..ebc9248c 100644
--- a/modules/sfp_skymem.py
+++ b/modules/sfp_skymem.py
@@ -103,7 +103,7 @@ class sfp_skymem(SpiderFootPlugin):
self.results[email] = True
# Loop through first 20 pages of results
- domain_ids = re.findall(r'<a href="/domain/([a-z0-9]+)\?p=', res['content'])
+ domain_ids = re.findall(r'<a href="/domain/([a-z0-9]+)\?p=', str(res['content']))
if not domain_ids:
return
@@ -136,7 +136,7 @@ class sfp_skymem(SpiderFootPlugin):
# Check if we're on the last page of results
max_page = 0
- pages = re.findall(r'/domain/' + domain_id + r'\?p=(\d+)', res['content'])
+ pages = re.findall(r'/domain/' + domain_id + r'\?p=(\d+)', str(res['content']))
for p in pages:
if int(p) >= max_page:
max_page = int(p)
diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.py
index 31ee8797..06292a9f 100644
--- a/modules/sfp_twitter.py
+++ b/modules/sfp_twitter.py
@@ -95,7 +95,7 @@ class sfp_twitter(SpiderFootPlugin):
# Retrieve name
human_name = re.findall(r'<div class="fullname">([^<]+)\s*</div>',
- res['content'], re.MULTILINE)
+ str(res['content']), re.MULTILINE)
if human_name:
e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0],