SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
SpiderFoot: validate fetchUrl() URL scheme
Brendan Colesa67aac13b280
diff --git a/sflib.py b/sflib.pyindex fd36167b..62687626 100644--- a/sflib.py+++ b/sflib.py@@ -2580,6 +2580,16 @@ class SpiderFoot:url = url.strip()+ try:+ parsed_url = urllib.parse.urlparse(url)+ except Exception:+ self.debug(f"Could not parse URL: {url}")+ return None++ if parsed_url.scheme != 'http' and parsed_url.scheme != 'https':+ self.debug(f"Invalid URL scheme for URL: {url}")+ return None+proxies = dict()if self.useProxyForUrl(url):proxy_url = f"socks5h://{self.opts['_socks2addr']}:{self.opts['_socks3port']}"diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 9a5c9615..4338d96f 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -1467,12 +1467,33 @@ class TestSpiderFoot(unittest.TestCase):"""sf = SpiderFoot(self.default_options)- invalid_types = [None, "", list(), dict(), int()]+ invalid_types = [None, list(), dict(), int()]for invalid_type in invalid_types:with self.subTest(invalid_type=invalid_type):res = sf.fetchUrl(invalid_type)self.assertEqual(None, res)+ def test_fetchUrl_argument_url_invalid_url_should_return_None(self):+ """+ Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,+ useragent="SpiderFoot", headers=None, noLog=False,+ postData=None, dontMangle=False, sizeLimit=None,+ headOnly=False, verify=False)+ """+ sf = SpiderFoot(self.default_options)++ res = sf.fetchUrl("")+ self.assertEqual(None, res)++ res = sf.fetchUrl("://spiderfoot.net/")+ self.assertEqual(None, res)++ res = sf.fetchUrl("file:///etc/hosts")+ self.assertEqual(None, res)++ res = sf.fetchUrl("irc://spiderfoot.net:6697/")+ self.assertEqual(None, res)+def test_check_dns_wildcard_invalid_target_should_return_none(self):"""Test checkDnsWildcard(self, target)