SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sflib: Rename fetchUrl() argument dontMangle to disableContentEncoding (#1337)
bcolesddc5cd844261
diff --git a/modules/sfp_binstring.py b/modules/sfp_binstring.pyindex e5dbb73b..884756dc 100644--- a/modules/sfp_binstring.py+++ b/modules/sfp_binstring.py@@ -130,7 +130,7 @@ class sfp_binstring(SpiderFootPlugin):res = self.sf.fetchUrl(eventData,useragent=self.opts['_useragent'],- dontMangle=True,+ disableContentEncoding=True,sizeLimit=self.opts['maxfilesize'],verify=False)diff --git a/modules/sfp_filemeta.py b/modules/sfp_filemeta.pyindex 4e2e2a12..4a634c68 100644--- a/modules/sfp_filemeta.py+++ b/modules/sfp_filemeta.py@@ -88,7 +88,7 @@ class sfp_filemeta(SpiderFootPlugin):# Fetch the file, allow much more time given that these files are# typically large.ret = self.sf.fetchUrl(eventData, timeout=self.opts['timeout'],- useragent=self.opts['_useragent'], dontMangle=True,+ useragent=self.opts['_useragent'], disableContentEncoding=True,sizeLimit=10000000,verify=False)if ret['content'] is None:diff --git a/sflib.py b/sflib.pyindex 74605d25..14ff7df8 100644--- a/sflib.py+++ b/sflib.py@@ -2187,7 +2187,7 @@ class SpiderFoot:headers: dict = None,noLog: bool = False,postData: str = None,- dontMangle: bool = False,+ disableContentEncoding: bool = False,sizeLimit: int = None,headOnly: bool = False,verify: bool = True@@ -2203,7 +2203,7 @@ class SpiderFoot:headers (dict): headersnoLog (bool): do not log requestpostData (str): HTTP POST data- dontMangle (bool): do not mangle+ disableContentEncoding (bool): do not UTF-8 encode response bodysizeLimit (int): size thresholdheadOnly (bool): use HTTP HEAD methodverify (bool): use HTTPS SSL/TLS verification@@ -2398,14 +2398,14 @@ class SpiderFoot:headers,noLog,postData,- dontMangle,+ disableContentEncoding,sizeLimit,headOnly)result['realurl'] = res.urlresult['code'] = str(res.status_code)- if dontMangle:+ if disableContentEncoding:result['content'] = res.contentelse:for encoding in ("utf-8", "ascii"):diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 69a016a9..7f09cb42 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -1304,7 +1304,7 @@ class TestSpiderFoot(unittest.TestCase):"""Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,useragent="SpiderFoot", headers=None, noLog=False,- postData=None, dontMangle=False, sizeLimit=None,+ postData=None, disableContentEncoding=False, sizeLimit=None,headOnly=False, verify=False)"""sf = SpiderFoot(self.default_options)@@ -1318,7 +1318,7 @@ class TestSpiderFoot(unittest.TestCase):"""Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,useragent="SpiderFoot", headers=None, noLog=False,- postData=None, dontMangle=False, sizeLimit=None,+ postData=None, disableContentEncoding=False, sizeLimit=None,headOnly=False, verify=False)"""sf = SpiderFoot(self.default_options)@@ -1332,7 +1332,7 @@ class TestSpiderFoot(unittest.TestCase):"""Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,useragent="SpiderFoot", headers=None, noLog=False,- postData=None, dontMangle=False, sizeLimit=None,+ postData=None, disableContentEncoding=False, sizeLimit=None,headOnly=False, verify=False)"""sf = SpiderFoot(self.default_options)@@ -1347,7 +1347,7 @@ class TestSpiderFoot(unittest.TestCase):"""Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,useragent="SpiderFoot", headers=None, noLog=False,- postData=None, dontMangle=False, sizeLimit=None,+ postData=None, disableContentEncoding=False, sizeLimit=None,headOnly=False, verify=False)"""sf = SpiderFoot(self.default_options)