Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

SpiderFoot: validate fetchUrl() URL scheme

Brendan Colesa67aac13b280
diff --git a/sflib.py b/sflib.py
index fd36167b..62687626 100644
--- a/sflib.py
+++ b/sflib.py
@@ -2580,6 +2580,16 @@ class SpiderFoot:
url = url.strip()
+ try:
+ parsed_url = urllib.parse.urlparse(url)
+ except Exception:
+ self.debug(f"Could not parse URL: {url}")
+ return None
+
+ if parsed_url.scheme != 'http' and parsed_url.scheme != 'https':
+ self.debug(f"Invalid URL scheme for URL: {url}")
+ return None
+
proxies = dict()
if self.useProxyForUrl(url):
proxy_url = f"socks5h://{self.opts['_socks2addr']}:{self.opts['_socks3port']}"
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index 9a5c9615..4338d96f 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -1467,12 +1467,33 @@ class TestSpiderFoot(unittest.TestCase):
"""
sf = SpiderFoot(self.default_options)
- invalid_types = [None, "", list(), dict(), int()]
+ invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
res = sf.fetchUrl(invalid_type)
self.assertEqual(None, res)
+ def test_fetchUrl_argument_url_invalid_url_should_return_None(self):
+ """
+ Test fetchUrl(self, url, fatal=False, cookies=None, timeout=30,
+ useragent="SpiderFoot", headers=None, noLog=False,
+ postData=None, dontMangle=False, sizeLimit=None,
+ headOnly=False, verify=False)
+ """
+ sf = SpiderFoot(self.default_options)
+
+ res = sf.fetchUrl("")
+ self.assertEqual(None, res)
+
+ res = sf.fetchUrl("://spiderfoot.net/")
+ self.assertEqual(None, res)
+
+ res = sf.fetchUrl("file:///etc/hosts")
+ self.assertEqual(None, res)
+
+ res = sf.fetchUrl("irc://spiderfoot.net:6697/")
+ self.assertEqual(None, res)
+
def test_check_dns_wildcard_invalid_target_should_return_none(self):
"""
Test checkDnsWildcard(self, target)