Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sflib: Rename fetchUrl() argument dontMangle to disableContentEncoding (#1337)

bcolesddc5cd844261
diff --git a/modules/sfp_binstring.py b/modules/sfp_binstring.py
index e5dbb73b..884756dc 100644
--- a/modules/sfp_binstring.py
+++ b/modules/sfp_binstring.py
@@ -130,7 +130,7 @@ class sfp_binstring(SpiderFootPlugin):
res = self.sf.fetchUrl(
eventData,
useragent=self.opts['_useragent'],
- dontMangle=True,
+ disableContentEncoding=True,
sizeLimit=self.opts['maxfilesize'],
verify=False
)
diff --git a/modules/sfp_filemeta.py b/modules/sfp_filemeta.py
index 4e2e2a12..4a634c68 100644
--- a/modules/sfp_filemeta.py
+++ b/modules/sfp_filemeta.py
@@ -88,7 +88,7 @@ class sfp_filemeta(SpiderFootPlugin):
# Fetch the file, allow much more time given that these files are
# typically large.
ret = self.sf.fetchUrl(eventData, timeout=self.opts['timeout'],
- useragent=self.opts['_useragent'], dontMangle=True,
+ useragent=self.opts['_useragent'], disableContentEncoding=True,
sizeLimit=10000000,
verify=False)
if ret['content'] is None:
diff --git a/sflib.py b/sflib.py
index 74605d25..14ff7df8 100644
--- a/sflib.py
+++ b/sflib.py
@@ -2187,7 +2187,7 @@ class SpiderFoot:
headers: dict = None,
noLog: bool = False,
postData: str = None,
- dontMangle: bool = False,
+ disableContentEncoding: bool = False,
sizeLimit: int = None,
headOnly: bool = False,
verify: bool = True
@@ -2203,7 +2203,7 @@ class SpiderFoot:
headers (dict): headers
noLog (bool): do not log request
postData (str): HTTP POST data
- dontMangle (bool): do not mangle
+ disableContentEncoding (bool): do not UTF-8 encode response body
sizeLimit (int): size threshold
headOnly (bool): use HTTP HEAD method
verify (bool): use HTTPS SSL/TLS verification
@@ -2398,14 +2398,14 @@ class SpiderFoot:
headers,
noLog,
postData,
- dontMangle,
+ disableContentEncoding,
sizeLimit,
headOnly
)
result['realurl'] = res.url
result['code'] = str(res.status_code)
- if dontMangle:
+ if disableContentEncoding:
result['content'] = res.content
else:
for encoding in ("utf-8", "ascii"):
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index 69a016a9..7f09cb42 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -1304,7 +1304,7 @@ class TestSpiderFoot(unittest.TestCase):
"""
Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,
useragent="SpiderFoot", headers=None, noLog=False,
- postData=None, dontMangle=False, sizeLimit=None,
+ postData=None, disableContentEncoding=False, sizeLimit=None,
headOnly=False, verify=False)
"""
sf = SpiderFoot(self.default_options)
@@ -1318,7 +1318,7 @@ class TestSpiderFoot(unittest.TestCase):
"""
Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,
useragent="SpiderFoot", headers=None, noLog=False,
- postData=None, dontMangle=False, sizeLimit=None,
+ postData=None, disableContentEncoding=False, sizeLimit=None,
headOnly=False, verify=False)
"""
sf = SpiderFoot(self.default_options)
@@ -1332,7 +1332,7 @@ class TestSpiderFoot(unittest.TestCase):
"""
Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,
useragent="SpiderFoot", headers=None, noLog=False,
- postData=None, dontMangle=False, sizeLimit=None,
+ postData=None, disableContentEncoding=False, sizeLimit=None,
headOnly=False, verify=False)
"""
sf = SpiderFoot(self.default_options)
@@ -1347,7 +1347,7 @@ class TestSpiderFoot(unittest.TestCase):
"""
Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,
useragent="SpiderFoot", headers=None, noLog=False,
- postData=None, dontMangle=False, sizeLimit=None,
+ postData=None, disableContentEncoding=False, sizeLimit=None,
headOnly=False, verify=False)
"""
sf = SpiderFoot(self.default_options)