Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

SpiderFootHelpers: extractLinksFromHtml: Use BeautifulSoup lxml parser (#1711)

bcolesb036b20c3cea
diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.py
index 94424fbb..14f8c004 100644
--- a/spiderfoot/helpers.py
+++ b/spiderfoot/helpers.py
@@ -770,7 +770,7 @@ class SpiderFootHelpers():
try:
for t in list(tags.keys()):
- for lnk in BeautifulSoup(data, features="xml", parse_only=SoupStrainer(t)).find_all(t):
+ for lnk in BeautifulSoup(data, features="lxml", parse_only=SoupStrainer(t)).find_all(t):
if lnk.has_attr(tags[t]):
links.append(lnk[tags[t]])
except BaseException:
diff --git a/test/unit/spiderfoot/test_spiderfoothelpers.py b/test/unit/spiderfoot/test_spiderfoothelpers.py
index 6917ee66..e83bd0b7 100644
--- a/test/unit/spiderfoot/test_spiderfoothelpers.py
+++ b/test/unit/spiderfoot/test_spiderfoothelpers.py
@@ -159,11 +159,11 @@ class TestSpiderFootHelpers(unittest.TestCase):
self.assertIsInstance(parse_links, dict)
self.assertFalse(parse_links)
- def test_extractLinksFromHtml_argument_data_containing_links_should_return_a_dict_of_urls(self):
+ def test_extractLinksFromHtml_argument_data_containing_malformed_html_with_links_should_return_a_dict_of_urls(self):
url = 'http://spiderfoot.net/'
parse_links = SpiderFootHelpers.extractLinksFromHtml(
url,
- '<html>example html content<a href="http://spiderfoot.net/path"></a><a href="/relative-path"></a></html>',
+ '<!DOCTYPE html><html lang="en-US"><meta charset="UTF-8" /><link rel="pingback" href="http://spiderfoot.net/xmlrpc.php">example html content<unclosed tag><a href="http://spiderfoot.net/path"></a><a href="/relative-path"></a></html>',
'domains'
)
self.assertIsInstance(parse_links, dict)
@@ -171,6 +171,7 @@ class TestSpiderFootHelpers(unittest.TestCase):
self.assertEqual(
parse_links,
{
+ 'http://spiderfoot.net/xmlrpc.php': {'source': url, 'original': 'http://spiderfoot.net/xmlrpc.php'},
'http://spiderfoot.net/path': {'source': url, 'original': 'http://spiderfoot.net/path'},
'http://spiderfoot.net/relative-path': {'source': url, 'original': '/relative-path'},
}