Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sflib: Move parseLinks to SpiderFootHelpers.extractLinksFromHtml() (#1683)

bcoles7f04a49d4780
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index c9b1df8e..42694350 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -132,8 +132,11 @@ class sfp_spider(SpiderFootPlugin):
url = fetched['realurl'] # override the URL if we had a redirect
# Extract links from the content
- links = self.sf.parseLinks(url, fetched['content'],
- self.getTarget().getNames())
+ links = SpiderFootHelpers.extractLinksFromHtml(
+ url,
+ fetched['content'],
+ self.getTarget().getNames()
+ )
if not links:
self.debug(f"No links found at {url}")
diff --git a/modules/sfp_wikileaks.py b/modules/sfp_wikileaks.py
index 164b7b73..bb6ce65e 100644
--- a/modules/sfp_wikileaks.py
+++ b/modules/sfp_wikileaks.py
@@ -12,7 +12,7 @@
import datetime
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
class sfp_wikileaks(SpiderFootPlugin):
@@ -72,11 +72,10 @@ class sfp_wikileaks(SpiderFootPlugin):
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
- srcModuleName = event.module
eventData = event.data
self.currentEventSrc = event
- self.debug(f"Received event, {eventName}, from {srcModuleName}")
+ self.debug(f"Received event, {eventName}, from {event.module}")
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
@@ -105,13 +104,12 @@ class sfp_wikileaks(SpiderFootPlugin):
self.error("Unable to fetch Wikileaks content.")
return
- # Fetch the paste site content
links = dict()
- p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org")
+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "wikileaks.org")
if p:
links.update(p)
- p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org")
+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "cryptome.org")
if p:
links.update(p)
@@ -126,23 +124,24 @@ class sfp_wikileaks(SpiderFootPlugin):
valid = False
for link in links:
+ # We can safely skip search.wikileaks.org and others.
if "search.wikileaks.org/" in link:
continue
- # We can safely skip search.wikileaks.org and others.
if "wikileaks.org/" not in link and "cryptome.org/" not in link:
continue
- else:
- self.debug("Found a link: " + link)
- if self.checkForStop():
- return
-
- # Wikileaks leak links will have a nested folder structure link
- if link.count('/') >= 4:
- if not link.endswith(".js") and not link.endswith(".css"):
- evt = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
- self.notifyListeners(evt)
- valid = True
+
+ self.debug(f"Found a link: {link}")
+
+ if self.checkForStop():
+ return
+
+ # Wikileaks leak links will have a nested folder structure link
+ if link.count('/') >= 4:
+ if not link.endswith(".js") and not link.endswith(".css"):
+ evt = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
+ self.notifyListeners(evt)
+ valid = True
if valid:
evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", res['content'],
@@ -160,16 +159,17 @@ class sfp_wikileaks(SpiderFootPlugin):
external + "&new_search=True&order_by=most_relevant&page=" + \
str(page) + "#results"
res = self.sf.fetchUrl(wlurl)
+ if not res:
+ break
if not res['content']:
break
- # Fetch the paste site content
links = dict()
- p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org")
+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "wikileaks.org")
if p:
links.update(p)
- p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org")
+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "cryptome.org")
if p:
links.update(p)
diff --git a/sflib.py b/sflib.py
index 49a982d9..ab13bc58 100644
--- a/sflib.py
+++ b/sflib.py
@@ -35,7 +35,6 @@ import netaddr
import OpenSSL
import requests
import urllib3
-from bs4 import BeautifulSoup, SoupStrainer
from publicsuffixlist import PublicSuffixList
from spiderfoot import SpiderFootHelpers
@@ -1159,126 +1158,6 @@ class SpiderFoot:
return ret
- def parseLinks(self, url: str, data: str, domains: list) -> dict:
- """Find all URLs within the supplied content.
-
- This function does not fetch any URLs.
-
- A dictionary will be returned, where each link will have the keys
- 'source': The URL where the link was obtained from
- 'original': What the link looked like in the content it was obtained from
- The key will be the *absolute* URL of the link obtained, so for example if
- the link '/abc' was obtained from 'http://xyz.com', the key in the dict will
- be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'
-
- Args:
- url (str): base URL used to construct absolute URLs from relative URLs
- data (str): data to examine for links
- domains: TBD
-
- Returns:
- dict: links
- """
- returnLinks = dict()
-
- if not isinstance(data, str):
- self.debug(f"parseLinks() data is {type(data)}; expected str()")
- return returnLinks
-
- if not data:
- self.debug("parseLinks() called with no data to parse.")
- return returnLinks
-
- if isinstance(domains, str):
- domains = [domains]
-
- tags = {
- 'a': 'href',
- 'img': 'src',
- 'script': 'src',
- 'link': 'href',
- 'area': 'href',
- 'base': 'href',
- 'form': 'action'
- }
-
- try:
- proto = url.split(":")[0]
- except BaseException:
- proto = "http"
- if proto is None:
- proto = "http"
-
- urlsRel = []
-
- try:
- for t in list(tags.keys()):
- for lnk in BeautifulSoup(data, features="xml", parse_only=SoupStrainer(t)).find_all(t):
- if lnk.has_attr(tags[t]):
- urlsRel.append(lnk[tags[t]])
- except BaseException as e:
- self.error(f"Error parsing with BeautifulSoup: {e}")
- return returnLinks
-
- # Loop through all the URLs/links found
- for link in urlsRel:
- if not isinstance(link, str):
- link = str(link)
- link = link.strip()
- linkl = link.lower()
- absLink = None
-
- if len(link) < 1:
- continue
-
- # Don't include stuff likely part of some dynamically built incomplete
- # URL found in Javascript code (character is part of some logic)
- if link[len(link) - 1] in ['.', '#'] or link[0] == '+' or 'javascript:' in linkl or '()' in link:
- self.debug('unlikely link: ' + link)
- continue
-
- # Filter in-page links
- if re.match('.*#.[^/]+', link):
- self.debug('in-page link: ' + link)
- continue
-
- # Ignore mail links
- if 'mailto:' in linkl:
- self.debug("Ignoring mail link: " + link)
- continue
-
- # URL decode links
- if '%2f' in linkl:
- link = urllib.parse.unquote(link)
-
- # Capture the absolute link:
- # If the link contains ://, it is already an absolute link
- if '://' in link:
- absLink = link
-
- # If the link starts with a /, the absolute link is off the base URL
- if link.startswith('/'):
- absLink = SpiderFootHelpers.urlBaseUrl(url) + link
-
- # Protocol relative URLs
- if link.startswith('//'):
- absLink = proto + ':' + link
-
- # Maybe the domain was just mentioned and not a link, so we make it one
- for domain in domains:
- if absLink is None and domain.lower() in link.lower():
- absLink = proto + '://' + link
-
- # Otherwise, it's a flat link within the current directory
- if absLink is None:
- absLink = SpiderFootHelpers.urlBaseDir(url) + link
-
- # Translate any relative pathing (../)
- absLink = SpiderFootHelpers.urlRelativeToAbsolute(absLink)
- returnLinks[absLink] = {'source': url, 'original': link}
-
- return returnLinks
-
def urlEncodeUnicode(self, url: str) -> str:
"""Encode a string as unicode.
@@ -1580,8 +1459,8 @@ class SpiderFoot:
timeout=timeout,
verify=verify
)
- except requests.exceptions.RequestException:
- self.error(f"Failed to connect to {url}")
+ except requests.exceptions.RequestException as e:
+ self.error(f"Failed to connect to {url}: {e}")
return result
except Exception as e:
if noLog:
diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.py
index f58d0e08..4f8a605f 100644
--- a/spiderfoot/helpers.py
+++ b/spiderfoot/helpers.py
@@ -5,10 +5,12 @@ import os
import os.path
import random
import re
+import urllib.parse
import uuid
from pathlib import Path
import networkx as nx
+from bs4 import BeautifulSoup, SoupStrainer
from networkx.readwrite.gexf import GEXFWriter
import phonenumbers
@@ -633,6 +635,126 @@ class SpiderFootHelpers():
"""
return str(uuid.uuid4()).split("-")[0].upper()
+ @staticmethod
+ def extractLinksFromHtml(url: str, data: str, domains: list) -> dict:
+ """Find all URLs within the supplied content.
+
+ This function does not fetch any URLs.
+
+ A dictionary will be returned, where each link will have the keys:
+ 'source': The URL where the link was obtained from
+ 'original': What the link looked like in the content it was obtained from
+
+ The key will be the *absolute* URL of the link obtained, so for example if
+ the link '/abc' was obtained from 'http://xyz.com', the key in the dict will
+ be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'
+
+ Args:
+ url (str): base URL used to construct absolute URLs from relative URLs
+ data (str): data to examine for links
+ domains: TBD
+
+ Returns:
+ dict: links
+
+ Raises:
+ TypeError: argument was invalid type
+ """
+ returnLinks = dict()
+
+ if not isinstance(url, str):
+ raise TypeError(f"url {type(url)}; expected str()")
+
+ if not isinstance(data, str):
+ raise TypeError(f"data {type(data)}; expected str()")
+
+ if isinstance(domains, str):
+ domains = [domains]
+
+ tags = {
+ 'a': 'href',
+ 'img': 'src',
+ 'script': 'src',
+ 'link': 'href',
+ 'area': 'href',
+ 'base': 'href',
+ 'form': 'action'
+ }
+
+ links = []
+
+ try:
+ for t in list(tags.keys()):
+ for lnk in BeautifulSoup(data, features="xml", parse_only=SoupStrainer(t)).find_all(t):
+ if lnk.has_attr(tags[t]):
+ links.append(lnk[tags[t]])
+ except BaseException:
+ return returnLinks
+
+ try:
+ proto = url.split(":")[0]
+ except BaseException:
+ proto = "http"
+ if proto is None:
+ proto = "http"
+
+ # Loop through all the URLs/links found
+ for link in links:
+ if not isinstance(link, str):
+ link = str(link)
+
+ link = link.strip()
+
+ if len(link) < 1:
+ continue
+
+ # Don't include stuff likely part of some dynamically built incomplete
+ # URL found in Javascript code (character is part of some logic)
+ if link[len(link) - 1] in ['.', '#'] or link[0] == '+' or 'javascript:' in link.lower() or '()' in link:
+ continue
+
+ # Filter in-page links
+ if re.match('.*#.[^/]+', link):
+ continue
+
+ # Ignore mail links
+ if 'mailto:' in link.lower():
+ continue
+
+ # URL decode links
+ if '%2f' in link.lower():
+ link = urllib.parse.unquote(link)
+
+ absLink = None
+
+ # Capture the absolute link:
+ # If the link contains ://, it is already an absolute link
+ if '://' in link:
+ absLink = link
+
+ # If the link starts with //, it is likely a protocol relative URL
+ elif link.startswith('//'):
+ absLink = proto + ':' + link
+
+ # If the link starts with a /, the absolute link is off the base URL
+ elif link.startswith('/'):
+ absLink = SpiderFootHelpers.urlBaseUrl(url) + link
+
+ # Maybe the domain was just mentioned and not a link, so we make it one
+ for domain in domains:
+ if absLink is None and domain.lower() in link.lower():
+ absLink = proto + '://' + link
+
+ # Otherwise, it's a flat link within the current directory
+ if absLink is None:
+ absLink = SpiderFootHelpers.urlBaseDir(url) + link
+
+ # Translate any relative pathing (../)
+ absLink = SpiderFootHelpers.urlRelativeToAbsolute(absLink)
+ returnLinks[absLink] = {'source': url, 'original': link}
+
+ return returnLinks
+
@staticmethod
def extractHashesFromText(data: str) -> list:
"""Extract all hashes within the supplied content.
diff --git a/test/unit/spiderfoot/test_spiderfoothelpers.py b/test/unit/spiderfoot/test_spiderfoothelpers.py
index 72480312..24cf25b2 100644
--- a/test/unit/spiderfoot/test_spiderfoothelpers.py
+++ b/test/unit/spiderfoot/test_spiderfoothelpers.py
@@ -172,6 +172,49 @@ class TestSpiderFootHelpers(unittest.TestCase):
scan_instance_id = SpiderFootHelpers.genScanInstanceId()
self.assertIsInstance(scan_instance_id, str)
+ def test_extractLinksFromHtml_argument_data_not_containing_links_should_return_an_empty_dict(self):
+ parse_links = SpiderFootHelpers.extractLinksFromHtml('url', '<html>example html content</html>', 'domains')
+ self.assertIsInstance(parse_links, dict)
+ self.assertFalse(parse_links)
+
+ def test_extractLinksFromHtml_argument_data_containing_links_should_return_a_dict_of_urls(self):
+ url = 'http://spiderfoot.net/'
+ parse_links = SpiderFootHelpers.extractLinksFromHtml(
+ url,
+ '<html>example html content<a href="http://spiderfoot.net/path"></a><a href="/relative-path"></a></html>',
+ 'domains'
+ )
+ self.assertIsInstance(parse_links, dict)
+
+ self.assertEqual(
+ parse_links,
+ {
+ 'http://spiderfoot.net/path': {'source': url, 'original': 'http://spiderfoot.net/path'},
+ 'http://spiderfoot.net/relative-path': {'source': url, 'original': '/relative-path'},
+ }
+ )
+
+ def test_extractLinksFromHtml_invalid_url_should_raise_TypeError(self):
+ invalid_types = [None, list(), dict()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ SpiderFootHelpers.extractLinksFromHtml(invalid_type, 'example html content', 'domains')
+
+ def test_extractLinksFromHtml_invalid_data_should_raise_TypeError(self):
+ invalid_types = [None, list(), dict()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ SpiderFootHelpers.extractLinksFromHtml("", invalid_type, 'domains')
+
+ def test_extractLinksFromHtml_invalid_domains_should_return_a_dict(self):
+ invalid_types = [None, "", list(), dict()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ parse_links = SpiderFootHelpers.extractLinksFromHtml('url', 'example html content', invalid_type)
+ self.assertIsInstance(parse_links, dict)
+
def test_extractHashesFromText_should_return_a_list(self):
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index 433924bb..ddd71130 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -812,56 +812,6 @@ class TestSpiderFoot(unittest.TestCase):
parse_cert = sf.parseCert(None, 'spiderfoot.net', invalid_type)
self.assertEqual(None, parse_cert)
- def test_parse_links_should_return_a_dict_of_urls_from_string(self):
- """
- Test parseLinks(self, url, data, domains)
- """
- sf = SpiderFoot(self.default_options)
-
- parse_links = sf.parseLinks('url', 'example html content', 'domains')
- self.assertIsInstance(parse_links, dict)
-
- parse_links = sf.parseLinks('http://spiderfoot.net/', 'example html content<a href="http://spiderfoot.net/path"></a>', 'domains')
- self.assertIsInstance(parse_links, dict)
-
- self.assertEqual('TBD', 'TBD')
-
- def test_parse_links_invalid_url_should_return_a_dict(self):
- """
- Test parseLinks(self, url, data, domains)
- """
- sf = SpiderFoot(self.default_options)
-
- invalid_types = [None, "", list(), dict()]
- for invalid_type in invalid_types:
- with self.subTest(invalid_type=invalid_type):
- parse_links = sf.parseLinks(invalid_type, 'example html content', 'domains')
- self.assertIsInstance(parse_links, dict)
-
- def test_parse_links_invalid_data_should_return_a_dict(self):
- """
- Test parseLinks(self, url, data, domains)
- """
- sf = SpiderFoot(self.default_options)
-
- invalid_types = [None, "", list(), dict()]
- for invalid_type in invalid_types:
- with self.subTest(invalid_type=invalid_type):
- parse_links = sf.parseLinks("", invalid_type, 'domains')
- self.assertIsInstance(parse_links, dict)
-
- def test_parse_links_invalid_domains_should_return_a_dict(self):
- """
- Test parseLinks(self, url, data, domains)
- """
- sf = SpiderFoot(self.default_options)
-
- invalid_types = [None, "", list(), dict()]
- for invalid_type in invalid_types:
- with self.subTest(invalid_type=invalid_type):
- parse_links = sf.parseLinks('url', 'example html content', invalid_type)
- self.assertIsInstance(parse_links, dict)
-
def test_url_encode_unicode_should_return_a_string(self):
"""
Test urlEncodeUnicode(self, url)