Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

- Removed old PublicSuffixList code. - Added need for publicsuffixlist lib & replaced code to use it. - Added tests. - Removed SpiderFootScanStatus and globalScanStatus code.

smicallef465f23a6776f
diff --git a/requirements.txt b/requirements.txt
index 7b44239a..3498cf9c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,3 +22,4 @@ python-docx>=0.8.10
python-pptx>=0.6.18
networkx==2.3
cryptography>=2.5
+publicsuffixlist>=0.7.3
diff --git a/sflib.py b/sflib.py
index e7bcd08f..9273895c 100644
--- a/sflib.py
+++ b/sflib.py
@@ -34,6 +34,7 @@ import OpenSSL
import uuid
import cryptography
import dns.resolver
+from publicsuffixlist import PublicSuffixList
from networkx import nx
from networkx.readwrite.gexf import GEXFWriter
from datetime import datetime
@@ -1064,8 +1065,8 @@ class SpiderFoot:
if not hostname:
return None
- ps = PublicSuffixList(tldList)
- return ps.get_public_suffix(hostname)
+ ps = PublicSuffixList(tldList, only_icann=True)
+ return ps.privatesuffix(hostname)
def validHost(self, hostname, tldList):
"""Check if the provided string is a valid hostname with a valid public suffix TLD.
@@ -1089,15 +1090,15 @@ class SpiderFoot:
if not re.match(r"^[a-z0-9-\.]*$", hostname, re.IGNORECASE):
return False
- ps = PublicSuffixList(tldList)
- sfx = ps.get_public_suffix(hostname, strict=True)
+ ps = PublicSuffixList(tldList, only_icann=True, accept_unknown=False)
+ sfx = ps.privatesuffix(hostname)
return sfx != None
def isDomain(self, hostname, tldList):
"""Check if the provided hostname string is a valid domain name.
Given a possible hostname, check if it's a domain name
- By checking whether it rests atop a TLD.
+ By checking whether it rests atop a valid TLD.
e.g. www.example.com = False because tld of hostname is com,
and www.example has a . in it.
@@ -1114,10 +1115,10 @@ class SpiderFoot:
if not hostname:
return False
- ps = PublicSuffixList(tldList)
- suffix = ps.get_public_suffix(hostname)
- return hostname == suffix
-
+ ps = PublicSuffixList(tldList, only_icann=True, accept_unknown=False)
+ sfx = ps.privatesuffix(hostname)
+ return sfx == hostname
+
def validIP(self, address):
"""Check if the provided string is a valid IPv4 address.
@@ -1972,10 +1973,10 @@ class SpiderFoot:
"""Remove key= and others from URLs to avoid credentials in logs."""
pats = {
- "key=\S+": "key=XXX",
- "pass=\S+": "pass=XXX",
- "user=\S+": "user=XXX",
- "password=\S+": "password=XXX"
+ r'key=\S+': "key=XXX",
+ r'pass=\S+': "pass=XXX",
+ r'user=\S+': "user=XXX",
+ r'password=\S+': "password=XXX"
}
ret = url
@@ -2728,146 +2729,3 @@ class SpiderFootEvent(object):
def setSourceEventHash(self, srcHash):
self.sourceEventHash = srcHash
-
-"""
-Public Suffix List module for Python.
-See LICENSE.tp for applicable license.
-"""
-
-
-class PublicSuffixList(object):
- def __init__(self, input_data):
- """Reads and parses public suffix list.
-
- input_file is a file object or another iterable that returns
- lines of a public suffix list file. If input_file is None, an
- UTF-8 encoded file named "publicsuffix.txt" in the same
- directory as this Python module is used.
-
- The file format is described at http://publicsuffix.org/list/
- """
-
- #if input_file is None:
- #input_path = os.path.join(os.path.dirname(__file__), 'publicsuffix.txt')
- #input_file = codecs.open(input_path, "r", "utf-8")
-
- root = self._build_structure(input_data)
- self.root = self._simplify(root)
-
- def _find_node(self, parent, parts):
- if not parts:
- return parent
-
- if len(parent) == 1:
- parent.append({})
-
- if len(parent) != 2:
- return None
-
- negate, children = parent
-
- child = parts.pop()
-
- child_node = children.get(child, None)
-
- if not child_node:
- children[child] = child_node = [0]
-
- return self._find_node(child_node, parts)
-
- def _add_rule(self, root, rule):
- if rule.startswith('!'):
- negate = 1
- rule = rule[1:]
- else:
- negate = 0
-
- parts = rule.split('.')
- self._find_node(root, parts)[0] = negate
-
- def _simplify(self, node):
- if len(node) == 1:
- return node[0]
-
- return (node[0], dict((k, self._simplify(v)) for (k, v) in list(node[1].items())))
-
- def _build_structure(self, fp):
- root = [0]
-
- for line in fp:
- line = str(line).strip()
- if line.startswith('//') or not line:
- continue
-
- self._add_rule(root, line.split()[0].lstrip('.'))
-
- return root
-
- def _lookup_node(self, matches, depth, parent, parts):
- if parent in (0, 1):
- negate = parent
- children = None
- else:
- negate, children = parent
-
- matches[-depth] = negate
-
- if depth < len(parts) and children:
- for name in ('*', parts[-depth]):
- child = children.get(name, None)
- if child is not None:
- self._lookup_node(matches, depth + 1, child, parts)
-
- def get_public_suffix(self, domain, strict=False):
- """get_public_suffix("www.example.com") -> "example.com"
-
- Calling this function with a DNS name will return the
- public suffix for that name.
-
- Note that for internationalized domains the list at
- http://publicsuffix.org uses decoded names, so it is
- up to the caller to decode any Punycode-encoded names.
- """
-
- parts = domain.lower().lstrip('.').split('.')
- hits = [None] * len(parts)
-
- if strict and parts[-1] not in self.root[1]:
- return None
-
- self._lookup_node(hits, 1, self.root, parts)
-
- for i, what in enumerate(hits):
- if what is not None and what == 0:
- return '.'.join(parts[i:])
-
-
-# Class for tracking the status of all running scans. Thread safe.
-class SpiderFootScanStatus:
- statusTable = dict()
- lock = threading.Lock()
-
- def setStatus(self, scanId, status):
- """Set scan status."""
-
- with self.lock:
- self.statusTable[scanId] = status
-
- def getStatus(self, scanId):
- """Get scan status."""
-
- with self.lock:
- if scanId in self.statusTable:
- return self.statusTable[scanId]
- return None
-
- def getStatusAll(self):
- """Set scan status for all scans."""
-
- with self.lock:
- return self.statusTable
-
-# Global variable accessed by various places to get the status of
-# running scans.
-globalScanStatus = SpiderFootScanStatus()
-
diff --git a/sfwebui.py b/sfwebui.py
index 22150fb6..a09c949f 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -857,7 +857,6 @@ class SpiderFootWebUi:
# Stop a scan (id variable is unnecessary for now given that only one simultaneous
# scan is permitted.)
def stopscanmulti(self, ids):
- #Removing All globalScanStatus Refs
dbh = SpiderFootDb(self.config)
error = list()
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index bb5e082f..a9baaca6 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -33,6 +33,8 @@ class TestSpiderFoot(unittest.TestCase):
'_torctlport': 9051,
'__logstdout': False
}
+
+ test_tlds = "// ===BEGIN ICANN DOMAINS===\n\ncom\nnet\norg\n\n// // ===END ICANN DOMAINS===\n"
def test_init_no_options(self):
"""
@@ -413,16 +415,18 @@ class TestSpiderFoot(unittest.TestCase):
Test domainKeyword(self, domain, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
keyword = sf.domainKeyword('www.spiderfoot.net', sf.opts.get('_internettlds'))
self.assertIsInstance(keyword, str)
- self.assertEqual('wwwspiderfootnet', keyword)
+ self.assertEqual('spiderfoot', keyword)
def test_domain_keyword_invalid_domain_should_return_none(self):
"""
Test domainKeyword(self, domain, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
keyword = sf.domainKeyword("", sf.opts.get('_internettlds'))
self.assertEqual(None, keyword)
@@ -436,18 +440,20 @@ class TestSpiderFoot(unittest.TestCase):
Test domainKeywords(self, domainList, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
domain_list = ['www.example.com', 'localhost.local']
keywords = sf.domainKeywords(domain_list, sf.opts.get('_internettlds'))
self.assertIsInstance(keywords, set)
- self.assertIn('localhostlocal', keywords)
- self.assertIn('wwwexamplecom', keywords)
+ self.assertIn('localhost', keywords)
+ self.assertIn('example', keywords)
def test_domain_keywords_invalid_domainlist_should_return_a_set(self):
"""
Test domainKeyword(self, domain, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
keywords = sf.domainKeywords("", sf.opts.get('_internettlds'))
self.assertIsInstance(keywords, set)
@@ -461,6 +467,7 @@ class TestSpiderFoot(unittest.TestCase):
Test hostDomain(self, hostname, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
host_domain = sf.hostDomain(None, sf.opts.get('_internettlds'))
self.assertEqual(None, host_domain)
@@ -470,18 +477,19 @@ class TestSpiderFoot(unittest.TestCase):
Test hostDomain(self, hostname, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
- host_domain = sf.hostDomain('local', sf.opts.get('_internettlds'))
+ host_domain = sf.hostDomain('www.spiderfoot.net', sf.opts.get('_internettlds'))
self.assertIsInstance(host_domain, str)
- self.assertEqual('local', host_domain)
+ self.assertEqual('spiderfoot.net', host_domain)
host_domain = sf.hostDomain('spiderfoot.net', sf.opts.get('_internettlds'))
self.assertIsInstance(host_domain, str)
- self.assertEqual('net', host_domain)
+ self.assertEqual('spiderfoot.net', host_domain)
- host_domain = sf.hostDomain('www.spiderfoot.net', sf.opts.get('_internettlds'))
+ host_domain = sf.hostDomain('abc.www.spiderfoot.net', sf.opts.get('_internettlds'))
self.assertIsInstance(host_domain, str)
- self.assertEqual('net', host_domain)
+ self.assertEqual('spiderfoot.net', host_domain)
def test_host_domain_invalid_tldlist_should_return_none(self):
"""
@@ -497,8 +505,9 @@ class TestSpiderFoot(unittest.TestCase):
Test isDomain(self, hostname, tldList)
"""
sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
- is_domain = sf.isDomain('local', sf.opts.get('_internettlds'))
+ is_domain = sf.isDomain('spiderfoot.net', sf.opts.get('_internettlds'))
self.assertIsInstance(is_domain, bool)
self.assertTrue(is_domain)
@@ -512,6 +521,49 @@ class TestSpiderFoot(unittest.TestCase):
self.assertIsInstance(is_domain, bool)
self.assertFalse(is_domain)
+ def test_is_domain_invalid_tld_should_return_false(self):
+ """
+ Test isDomain(self, hostname, tldList)
+ """
+ sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
+
+ is_domain = sf.isDomain('spiderfoot.not_a_tld', sf.opts.get('_internettlds'))
+ self.assertIsInstance(is_domain, bool)
+ self.assertFalse(is_domain)
+
+ def test_valid_host_invalid_tldlist_should_return_false(self):
+ """
+ Test validHost(self, hostname, tldList)
+ """
+ sf = SpiderFoot(self.default_options)
+
+ is_host = sf.validHost('spiderfoot.net', None)
+ self.assertIsInstance(is_host, bool)
+ self.assertFalse(is_host)
+
+ def test_valid_host_valid_host_should_return_true(self):
+ """
+ Test validHost(self, hostname, tldList)
+ """
+ sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
+
+ is_host = sf.validHost('spiderfoot.net', sf.opts.get('_internettlds'))
+ self.assertIsInstance(is_host, bool)
+ self.assertTrue(is_host)
+
+ def test_valid_host_invalid_host_should_return_false(self):
+ """
+ Test validHost(self, hostname, tldList)
+ """
+ sf = SpiderFoot(self.default_options)
+ sf.opts['_internettlds'] = self.test_tlds
+
+ is_valid = sf.validHost('something.gif', sf.opts.get('_internettlds'))
+ self.assertIsInstance(is_valid, bool)
+ self.assertFalse(is_valid)
+
def test_valid_ip_should_return_a_boolean(self):
"""
Test validIP(self, address)