SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
modules: cast fetchUrl content as string in calls to findall() (#1339)
bcoles8213ee5c57d3
diff --git a/modules/sfp_callername.py b/modules/sfp_callername.pyindex 9aa7b85f..1e290faf 100644--- a/modules/sfp_callername.py+++ b/modules/sfp_callername.py@@ -110,7 +110,7 @@ class sfp_callername(SpiderFootPlugin):self.sf.debug('No phone information found for ' + eventData)return- location_match = re.findall(r'<div class="callerid"><h4>.*?</h4><p>(.+?)</p></div>', res['content'], re.MULTILINE | re.DOTALL)+ location_match = re.findall(r'<div class="callerid"><h4>.*?</h4><p>(.+?)</p></div>', str(res['content']), re.MULTILINE | re.DOTALL)if location_match:location = location_match[0]@@ -121,8 +121,8 @@ class sfp_callername(SpiderFootPlugin):evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)self.notifyListeners(evt)- rep_good_match = re.findall(r'>SAFE.*?>(\d+) votes?<', res['content'])- rep_bad_match = re.findall(r'>UNSAFE.*?>(\d+) votes?<', res['content'])+ rep_good_match = re.findall(r'>SAFE.*?>(\d+) votes?<', str(res['content']))+ rep_bad_match = re.findall(r'>UNSAFE.*?>(\d+) votes?<', str(res['content']))if rep_good_match and rep_bad_match:good_votes = int(rep_good_match[0])diff --git a/modules/sfp_commoncrawl.py b/modules/sfp_commoncrawl.pyindex 74ce3e44..394a74d1 100644--- a/modules/sfp_commoncrawl.py+++ b/modules/sfp_commoncrawl.py@@ -101,7 +101,7 @@ class sfp_commoncrawl(SpiderFootPlugin):self.errorState = Truereturn list()- indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", res['content'])+ indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", str(res['content']))indexlist = dict()for m in indexes:ms = m.replace("CC-MAIN-", "").replace("-", "")diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.pyindex 430d838b..ed9798fb 100644--- a/modules/sfp_crossref.py+++ b/modules/sfp_crossref.py@@ -113,7 +113,7 @@ class sfp_crossref(SpiderFootPlugin):r"([\.\'\/\"\ ]" + re.escape(name) + r"[\.\'\/\"\ ])",re.IGNORECASE)- matches = re.findall(pat, res['content'])+ matches = re.findall(pat, str(res['content']))if len(matches) > 0:matched = True@@ -144,7 +144,7 @@ class sfp_crossref(SpiderFootPlugin):r"([\.\'\/\"\ ]" + re.escape(name) + r"[\'\/\"\ ])",re.IGNORECASE)- matches = re.findall(pat, res['content'])+ matches = re.findall(pat, str(res['content']))if len(matches) > 0:matched = Truediff --git a/modules/sfp_flickr.py b/modules/sfp_flickr.pyindex 9a7318eb..8c78578f 100644--- a/modules/sfp_flickr.py+++ b/modules/sfp_flickr.py@@ -91,7 +91,7 @@ class sfp_flickr(SpiderFootPlugin):if res['content'] is None:return None- keys = re.findall(r'YUI_config.flickr.api.site_key = "([a-zA-Z0-9]+)"', res['content'])+ keys = re.findall(r'YUI_config.flickr.api.site_key = "([a-zA-Z0-9]+)"', str(res['content']))if not keys:return Nonediff --git a/modules/sfp_h1nobbdde.py b/modules/sfp_h1nobbdde.pyindex 89b0fdff..9cca1c40 100644--- a/modules/sfp_h1nobbdde.py+++ b/modules/sfp_h1nobbdde.py@@ -77,7 +77,7 @@ class sfp_h1nobbdde(SpiderFootPlugin):try:rx = re.compile("<a class=\"title\" href=.(.[^\"]+).*?title=.(.[^\"\']+)", re.IGNORECASE | re.DOTALL)- for m in rx.findall(res['content']):+ for m in rx.findall(str(res['content'])):# Report itif qry in m[1]:ret.append(m[1] + "\n<SFURL>" + m[0] + "</SFURL>")diff --git a/modules/sfp_hackertarget.py b/modules/sfp_hackertarget.pyindex e96a8072..e74390e8 100644--- a/modules/sfp_hackertarget.py+++ b/modules/sfp_hackertarget.py@@ -130,7 +130,7 @@ class sfp_hackertarget(SpiderFootPlugin):if res['content'] is None:return None- html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', res['content'], re.MULTILINE | re.DOTALL)+ html_data = re.findall(r'<pre id="formResponse">(.*?)</pre>', str(res['content']), re.MULTILINE | re.DOTALL)if not html_data:self.sf.debug(f"Found no open UDP ports on {ip}")diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.pyindex 791bd315..9cbb57e2 100644--- a/modules/sfp_myspace.py+++ b/modules/sfp_myspace.py@@ -84,7 +84,7 @@ class sfp_myspace(SpiderFootPlugin):return# Extract HTML containing potential profile matches- profiles = re.findall(r'<a href="/[a-zA-Z0-9_]+">[^<]+</a></h6>', res['content'])+ profiles = re.findall(r'<a href="/[a-zA-Z0-9_]+">[^<]+</a></h6>', str(res['content']))if not profiles:self.sf.debug(f"No profiles found for e-mail: {email}")diff --git a/modules/sfp_open_passive_dns_database.py b/modules/sfp_open_passive_dns_database.pyindex 4cc9b6a2..0d784911 100644--- a/modules/sfp_open_passive_dns_database.py+++ b/modules/sfp_open_passive_dns_database.py@@ -93,7 +93,7 @@ class sfp_open_passive_dns_database(SpiderFootPlugin):self.sf.debug("No results found for " + qry)return None- rows = re.findall(r'<tr>(.+?)</tr>', res['content'], re.DOTALL)+ rows = re.findall(r'<tr>(.+?)</tr>', str(res['content']), re.DOTALL)if not rows:self.sf.debug("No passive DNS results for " + qry)diff --git a/modules/sfp_openbugbounty.py b/modules/sfp_openbugbounty.pyindex de73c2b2..17a0a609 100644--- a/modules/sfp_openbugbounty.py+++ b/modules/sfp_openbugbounty.py@@ -84,7 +84,7 @@ class sfp_openbugbounty(SpiderFootPlugin):try:rx = re.compile(".*<div class=.cell1.><a href=.(.*).>(.*" + qry + ").*?</a></div>.*", re.IGNORECASE)- for m in rx.findall(res['content']):+ for m in rx.findall(str(res['content'])):# Report itif m[1] == qry or m[1].endswith("." + qry):ret.append("From openbugbounty.org: <SFURL>" + base + m[0] + "</SFURL>")diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.pyindex 6ecf5760..af4f3cb5 100644--- a/modules/sfp_pgp.py+++ b/modules/sfp_pgp.py@@ -115,7 +115,7 @@ class sfp_pgp(SpiderFootPlugin):if res['content'] is not None and res['code'] != "503":pat = re.compile("(-----BEGIN.*END.*BLOCK-----)", re.MULTILINE | re.DOTALL)- matches = re.findall(pat, res['content'])+ matches = re.findall(pat, str(res['content']))for match in matches:self.sf.debug("Found public key: " + match)if len(match) < 300:diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.pyindex d5eac401..ebc9248c 100644--- a/modules/sfp_skymem.py+++ b/modules/sfp_skymem.py@@ -103,7 +103,7 @@ class sfp_skymem(SpiderFootPlugin):self.results[email] = True# Loop through first 20 pages of results- domain_ids = re.findall(r'<a href="/domain/([a-z0-9]+)\?p=', res['content'])+ domain_ids = re.findall(r'<a href="/domain/([a-z0-9]+)\?p=', str(res['content']))if not domain_ids:return@@ -136,7 +136,7 @@ class sfp_skymem(SpiderFootPlugin):# Check if we're on the last page of resultsmax_page = 0- pages = re.findall(r'/domain/' + domain_id + r'\?p=(\d+)', res['content'])+ pages = re.findall(r'/domain/' + domain_id + r'\?p=(\d+)', str(res['content']))for p in pages:if int(p) >= max_page:max_page = int(p)diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.pyindex 31ee8797..06292a9f 100644--- a/modules/sfp_twitter.py+++ b/modules/sfp_twitter.py@@ -95,7 +95,7 @@ class sfp_twitter(SpiderFootPlugin):# Retrieve namehuman_name = re.findall(r'<div class="fullname">([^<]+)\s*</div>',- res['content'], re.MULTILINE)+ str(res['content']), re.MULTILINE)if human_name:e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0],