SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sflib: return None if not url
Brendan Colesdf6b5f5807ac
diff --git a/sflib.py b/sflib.pyindex 1c113111..48c2aee8 100644--- a/sflib.py+++ b/sflib.py@@ -2455,6 +2455,9 @@ class SpiderFoot:headOnly=False, verify=True):"""Fetch a URL, return the response object."""+ if not url:+ return None+result = {'code': None,'status': None,@@ -2463,9 +2466,6 @@ class SpiderFoot:'realurl': url}- if url is None:- return None-url = url.strip()proxies = dict()diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 55b77c36..38b7e583 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -1414,8 +1414,7 @@ class TestSpiderFoot(unittest.TestCase):new_url = sf.removeUrlCreds(url)self.assertNotIn("secret", new_url)- @unittest.skip("todo")- def test_fetch_url(self):+ def test_fetchUrl_argument_url_should_return_http_response_as_dict(self):"""Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,useragent="SpiderFoot", headers=None, noLog=False,@@ -1424,13 +1423,26 @@ class TestSpiderFoot(unittest.TestCase):"""sf = SpiderFoot(self.default_options)- data = sf.fetchUrl(None)- self.assertEqual(data, None)+ res = sf.fetchUrl("https://spiderfoot.net/")+ self.assertIsInstance(res, dict)+ self.assertEqual(res['code'], "200")+ self.assertNotEqual(res['content'], None)- self.assertEqual('TBD', 'TBD')+ def test_fetchUrl_argument_headOnly_should_return_http_response_as_dict(self):+ """+ Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,+ useragent="SpiderFoot", headers=None, noLog=False,+ postData=None, dontMangle=False, sizeLimit=None,+ headOnly=False, verify=False)+ """+ sf = SpiderFoot(self.default_options)- @unittest.skip("todo")- def test_fetch_url_invalid_url_should_return_none(self):+ res = sf.fetchUrl("https://spiderfoot.net/", headOnly=True)+ self.assertIsInstance(res, dict)+ self.assertEqual(res['code'], "301")+ self.assertEqual(res['content'], None)++ def test_fetchUrl_argument_url_invalid_type_should_return_none(self):"""Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,useragent="SpiderFoot", headers=None, noLog=False,@@ -1439,7 +1451,7 @@ class TestSpiderFoot(unittest.TestCase):"""sf = SpiderFoot(self.default_options)- invalid_types = [None, "", list(), dict()]+ invalid_types = [None, "", list(), dict(), int()]for invalid_type in invalid_types:with self.subTest(invalid_type=invalid_type):res = sf.fetchUrl(invalid_type)