SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sflib: Move parseLinks to SpiderFootHelpers.extractLinksFromHtml() (#1683)
bcoles7f04a49d4780
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex c9b1df8e..42694350 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -132,8 +132,11 @@ class sfp_spider(SpiderFootPlugin):url = fetched['realurl'] # override the URL if we had a redirect# Extract links from the content- links = self.sf.parseLinks(url, fetched['content'],- self.getTarget().getNames())+ links = SpiderFootHelpers.extractLinksFromHtml(+ url,+ fetched['content'],+ self.getTarget().getNames()+ )if not links:self.debug(f"No links found at {url}")diff --git a/modules/sfp_wikileaks.py b/modules/sfp_wikileaks.pyindex 164b7b73..bb6ce65e 100644--- a/modules/sfp_wikileaks.py+++ b/modules/sfp_wikileaks.py@@ -12,7 +12,7 @@import datetime-from spiderfoot import SpiderFootEvent, SpiderFootPlugin+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPluginclass sfp_wikileaks(SpiderFootPlugin):@@ -72,11 +72,10 @@ class sfp_wikileaks(SpiderFootPlugin):# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventType- srcModuleName = event.moduleeventData = event.dataself.currentEventSrc = event- self.debug(f"Received event, {eventName}, from {srcModuleName}")+ self.debug(f"Received event, {eventName}, from {event.module}")if eventData in self.results:self.debug(f"Skipping {eventData}, already checked.")@@ -105,13 +104,12 @@ class sfp_wikileaks(SpiderFootPlugin):self.error("Unable to fetch Wikileaks content.")return- # Fetch the paste site contentlinks = dict()- p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org")+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "wikileaks.org")if p:links.update(p)- p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org")+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "cryptome.org")if p:links.update(p)@@ -126,23 +124,24 @@ class sfp_wikileaks(SpiderFootPlugin):valid = Falsefor link in links:+ # We can safely skip search.wikileaks.org and others.if "search.wikileaks.org/" in link:continue- # We can safely skip search.wikileaks.org and others.if "wikileaks.org/" not in link and "cryptome.org/" not in link:continue- else:- self.debug("Found a link: " + link)- if self.checkForStop():- return-- # Wikileaks leak links will have a nested folder structure link- if link.count('/') >= 4:- if not link.endswith(".js") and not link.endswith(".css"):- evt = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)- self.notifyListeners(evt)- valid = True++ self.debug(f"Found a link: {link}")++ if self.checkForStop():+ return++ # Wikileaks leak links will have a nested folder structure link+ if link.count('/') >= 4:+ if not link.endswith(".js") and not link.endswith(".css"):+ evt = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)+ self.notifyListeners(evt)+ valid = Trueif valid:evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", res['content'],@@ -160,16 +159,17 @@ class sfp_wikileaks(SpiderFootPlugin):external + "&new_search=True&order_by=most_relevant&page=" + \str(page) + "#results"res = self.sf.fetchUrl(wlurl)+ if not res:+ breakif not res['content']:break- # Fetch the paste site contentlinks = dict()- p = self.sf.parseLinks(wlurl, res['content'], "wikileaks.org")+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "wikileaks.org")if p:links.update(p)- p = self.sf.parseLinks(wlurl, res['content'], "cryptome.org")+ p = SpiderFootHelpers.extractLinksFromHtml(wlurl, res['content'], "cryptome.org")if p:links.update(p)diff --git a/sflib.py b/sflib.pyindex 49a982d9..ab13bc58 100644--- a/sflib.py+++ b/sflib.py@@ -35,7 +35,6 @@ import netaddrimport OpenSSLimport requestsimport urllib3-from bs4 import BeautifulSoup, SoupStrainerfrom publicsuffixlist import PublicSuffixListfrom spiderfoot import SpiderFootHelpers@@ -1159,126 +1158,6 @@ class SpiderFoot:return ret- def parseLinks(self, url: str, data: str, domains: list) -> dict:- """Find all URLs within the supplied content.-- This function does not fetch any URLs.-- A dictionary will be returned, where each link will have the keys- 'source': The URL where the link was obtained from- 'original': What the link looked like in the content it was obtained from- The key will be the *absolute* URL of the link obtained, so for example if- the link '/abc' was obtained from 'http://xyz.com', the key in the dict will- be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'-- Args:- url (str): base URL used to construct absolute URLs from relative URLs- data (str): data to examine for links- domains: TBD-- Returns:- dict: links- """- returnLinks = dict()-- if not isinstance(data, str):- self.debug(f"parseLinks() data is {type(data)}; expected str()")- return returnLinks-- if not data:- self.debug("parseLinks() called with no data to parse.")- return returnLinks-- if isinstance(domains, str):- domains = [domains]-- tags = {- 'a': 'href',- 'img': 'src',- 'script': 'src',- 'link': 'href',- 'area': 'href',- 'base': 'href',- 'form': 'action'- }-- try:- proto = url.split(":")[0]- except BaseException:- proto = "http"- if proto is None:- proto = "http"-- urlsRel = []-- try:- for t in list(tags.keys()):- for lnk in BeautifulSoup(data, features="xml", parse_only=SoupStrainer(t)).find_all(t):- if lnk.has_attr(tags[t]):- urlsRel.append(lnk[tags[t]])- except BaseException as e:- self.error(f"Error parsing with BeautifulSoup: {e}")- return returnLinks-- # Loop through all the URLs/links found- for link in urlsRel:- if not isinstance(link, str):- link = str(link)- link = link.strip()- linkl = link.lower()- absLink = None-- if len(link) < 1:- continue-- # Don't include stuff likely part of some dynamically built incomplete- # URL found in Javascript code (character is part of some logic)- if link[len(link) - 1] in ['.', '#'] or link[0] == '+' or 'javascript:' in linkl or '()' in link:- self.debug('unlikely link: ' + link)- continue-- # Filter in-page links- if re.match('.*#.[^/]+', link):- self.debug('in-page link: ' + link)- continue-- # Ignore mail links- if 'mailto:' in linkl:- self.debug("Ignoring mail link: " + link)- continue-- # URL decode links- if '%2f' in linkl:- link = urllib.parse.unquote(link)-- # Capture the absolute link:- # If the link contains ://, it is already an absolute link- if '://' in link:- absLink = link-- # If the link starts with a /, the absolute link is off the base URL- if link.startswith('/'):- absLink = SpiderFootHelpers.urlBaseUrl(url) + link-- # Protocol relative URLs- if link.startswith('//'):- absLink = proto + ':' + link-- # Maybe the domain was just mentioned and not a link, so we make it one- for domain in domains:- if absLink is None and domain.lower() in link.lower():- absLink = proto + '://' + link-- # Otherwise, it's a flat link within the current directory- if absLink is None:- absLink = SpiderFootHelpers.urlBaseDir(url) + link-- # Translate any relative pathing (../)- absLink = SpiderFootHelpers.urlRelativeToAbsolute(absLink)- returnLinks[absLink] = {'source': url, 'original': link}-- return returnLinks-def urlEncodeUnicode(self, url: str) -> str:"""Encode a string as unicode.@@ -1580,8 +1459,8 @@ class SpiderFoot:timeout=timeout,verify=verify)- except requests.exceptions.RequestException:- self.error(f"Failed to connect to {url}")+ except requests.exceptions.RequestException as e:+ self.error(f"Failed to connect to {url}: {e}")return resultexcept Exception as e:if noLog:diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.pyindex f58d0e08..4f8a605f 100644--- a/spiderfoot/helpers.py+++ b/spiderfoot/helpers.py@@ -5,10 +5,12 @@ import osimport os.pathimport randomimport re+import urllib.parseimport uuidfrom pathlib import Pathimport networkx as nx+from bs4 import BeautifulSoup, SoupStrainerfrom networkx.readwrite.gexf import GEXFWriterimport phonenumbers@@ -633,6 +635,126 @@ class SpiderFootHelpers():"""return str(uuid.uuid4()).split("-")[0].upper()+ @staticmethod+ def extractLinksFromHtml(url: str, data: str, domains: list) -> dict:+ """Find all URLs within the supplied content.++ This function does not fetch any URLs.++ A dictionary will be returned, where each link will have the keys:+ 'source': The URL where the link was obtained from+ 'original': What the link looked like in the content it was obtained from++ The key will be the *absolute* URL of the link obtained, so for example if+ the link '/abc' was obtained from 'http://xyz.com', the key in the dict will+ be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'++ Args:+ url (str): base URL used to construct absolute URLs from relative URLs+ data (str): data to examine for links+ domains: TBD++ Returns:+ dict: links++ Raises:+ TypeError: argument was invalid type+ """+ returnLinks = dict()++ if not isinstance(url, str):+ raise TypeError(f"url {type(url)}; expected str()")++ if not isinstance(data, str):+ raise TypeError(f"data {type(data)}; expected str()")++ if isinstance(domains, str):+ domains = [domains]++ tags = {+ 'a': 'href',+ 'img': 'src',+ 'script': 'src',+ 'link': 'href',+ 'area': 'href',+ 'base': 'href',+ 'form': 'action'+ }++ links = []++ try:+ for t in list(tags.keys()):+ for lnk in BeautifulSoup(data, features="xml", parse_only=SoupStrainer(t)).find_all(t):+ if lnk.has_attr(tags[t]):+ links.append(lnk[tags[t]])+ except BaseException:+ return returnLinks++ try:+ proto = url.split(":")[0]+ except BaseException:+ proto = "http"+ if proto is None:+ proto = "http"++ # Loop through all the URLs/links found+ for link in links:+ if not isinstance(link, str):+ link = str(link)++ link = link.strip()++ if len(link) < 1:+ continue++ # Don't include stuff likely part of some dynamically built incomplete+ # URL found in Javascript code (character is part of some logic)+ if link[len(link) - 1] in ['.', '#'] or link[0] == '+' or 'javascript:' in link.lower() or '()' in link:+ continue++ # Filter in-page links+ if re.match('.*#.[^/]+', link):+ continue++ # Ignore mail links+ if 'mailto:' in link.lower():+ continue++ # URL decode links+ if '%2f' in link.lower():+ link = urllib.parse.unquote(link)++ absLink = None++ # Capture the absolute link:+ # If the link contains ://, it is already an absolute link+ if '://' in link:+ absLink = link++ # If the link starts with //, it is likely a protocol relative URL+ elif link.startswith('//'):+ absLink = proto + ':' + link++ # If the link starts with a /, the absolute link is off the base URL+ elif link.startswith('/'):+ absLink = SpiderFootHelpers.urlBaseUrl(url) + link++ # Maybe the domain was just mentioned and not a link, so we make it one+ for domain in domains:+ if absLink is None and domain.lower() in link.lower():+ absLink = proto + '://' + link++ # Otherwise, it's a flat link within the current directory+ if absLink is None:+ absLink = SpiderFootHelpers.urlBaseDir(url) + link++ # Translate any relative pathing (../)+ absLink = SpiderFootHelpers.urlRelativeToAbsolute(absLink)+ returnLinks[absLink] = {'source': url, 'original': link}++ return returnLinks+@staticmethoddef extractHashesFromText(data: str) -> list:"""Extract all hashes within the supplied content.diff --git a/test/unit/spiderfoot/test_spiderfoothelpers.py b/test/unit/spiderfoot/test_spiderfoothelpers.pyindex 72480312..24cf25b2 100644--- a/test/unit/spiderfoot/test_spiderfoothelpers.py+++ b/test/unit/spiderfoot/test_spiderfoothelpers.py@@ -172,6 +172,49 @@ class TestSpiderFootHelpers(unittest.TestCase):scan_instance_id = SpiderFootHelpers.genScanInstanceId()self.assertIsInstance(scan_instance_id, str)+ def test_extractLinksFromHtml_argument_data_not_containing_links_should_return_an_empty_dict(self):+ parse_links = SpiderFootHelpers.extractLinksFromHtml('url', '<html>example html content</html>', 'domains')+ self.assertIsInstance(parse_links, dict)+ self.assertFalse(parse_links)++ def test_extractLinksFromHtml_argument_data_containing_links_should_return_a_dict_of_urls(self):+ url = 'http://spiderfoot.net/'+ parse_links = SpiderFootHelpers.extractLinksFromHtml(+ url,+ '<html>example html content<a href="http://spiderfoot.net/path"></a><a href="/relative-path"></a></html>',+ 'domains'+ )+ self.assertIsInstance(parse_links, dict)++ self.assertEqual(+ parse_links,+ {+ 'http://spiderfoot.net/path': {'source': url, 'original': 'http://spiderfoot.net/path'},+ 'http://spiderfoot.net/relative-path': {'source': url, 'original': '/relative-path'},+ }+ )++ def test_extractLinksFromHtml_invalid_url_should_raise_TypeError(self):+ invalid_types = [None, list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ SpiderFootHelpers.extractLinksFromHtml(invalid_type, 'example html content', 'domains')++ def test_extractLinksFromHtml_invalid_data_should_raise_TypeError(self):+ invalid_types = [None, list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ SpiderFootHelpers.extractLinksFromHtml("", invalid_type, 'domains')++ def test_extractLinksFromHtml_invalid_domains_should_return_a_dict(self):+ invalid_types = [None, "", list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ parse_links = SpiderFootHelpers.extractLinksFromHtml('url', 'example html content', invalid_type)+ self.assertIsInstance(parse_links, dict)+def test_extractHashesFromText_should_return_a_list(self):invalid_types = [None, list(), dict(), int()]for invalid_type in invalid_types:diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 433924bb..ddd71130 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -812,56 +812,6 @@ class TestSpiderFoot(unittest.TestCase):parse_cert = sf.parseCert(None, 'spiderfoot.net', invalid_type)self.assertEqual(None, parse_cert)- def test_parse_links_should_return_a_dict_of_urls_from_string(self):- """- Test parseLinks(self, url, data, domains)- """- sf = SpiderFoot(self.default_options)-- parse_links = sf.parseLinks('url', 'example html content', 'domains')- self.assertIsInstance(parse_links, dict)-- parse_links = sf.parseLinks('http://spiderfoot.net/', 'example html content<a href="http://spiderfoot.net/path"></a>', 'domains')- self.assertIsInstance(parse_links, dict)-- self.assertEqual('TBD', 'TBD')-- def test_parse_links_invalid_url_should_return_a_dict(self):- """- Test parseLinks(self, url, data, domains)- """- sf = SpiderFoot(self.default_options)-- invalid_types = [None, "", list(), dict()]- for invalid_type in invalid_types:- with self.subTest(invalid_type=invalid_type):- parse_links = sf.parseLinks(invalid_type, 'example html content', 'domains')- self.assertIsInstance(parse_links, dict)-- def test_parse_links_invalid_data_should_return_a_dict(self):- """- Test parseLinks(self, url, data, domains)- """- sf = SpiderFoot(self.default_options)-- invalid_types = [None, "", list(), dict()]- for invalid_type in invalid_types:- with self.subTest(invalid_type=invalid_type):- parse_links = sf.parseLinks("", invalid_type, 'domains')- self.assertIsInstance(parse_links, dict)-- def test_parse_links_invalid_domains_should_return_a_dict(self):- """- Test parseLinks(self, url, data, domains)- """- sf = SpiderFoot(self.default_options)-- invalid_types = [None, "", list(), dict()]- for invalid_type in invalid_types:- with self.subTest(invalid_type=invalid_type):- parse_links = sf.parseLinks('url', 'example html content', invalid_type)- self.assertIsInstance(parse_links, dict)-def test_url_encode_unicode_should_return_a_string(self):"""Test urlEncodeUnicode(self, url)