SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
- Removed old PublicSuffixList code. - Added need for publicsuffixlist lib & replaced code to use it. - Added tests. - Removed SpiderFootScanStatus and globalScanStatus code.
smicallef465f23a6776f
diff --git a/requirements.txt b/requirements.txtindex 7b44239a..3498cf9c 100644--- a/requirements.txt+++ b/requirements.txt@@ -22,3 +22,4 @@ python-docx>=0.8.10python-pptx>=0.6.18networkx==2.3cryptography>=2.5+publicsuffixlist>=0.7.3diff --git a/sflib.py b/sflib.pyindex e7bcd08f..9273895c 100644--- a/sflib.py+++ b/sflib.py@@ -34,6 +34,7 @@ import OpenSSLimport uuidimport cryptographyimport dns.resolver+from publicsuffixlist import PublicSuffixListfrom networkx import nxfrom networkx.readwrite.gexf import GEXFWriterfrom datetime import datetime@@ -1064,8 +1065,8 @@ class SpiderFoot:if not hostname:return None- ps = PublicSuffixList(tldList)- return ps.get_public_suffix(hostname)+ ps = PublicSuffixList(tldList, only_icann=True)+ return ps.privatesuffix(hostname)def validHost(self, hostname, tldList):"""Check if the provided string is a valid hostname with a valid public suffix TLD.@@ -1089,15 +1090,15 @@ class SpiderFoot:if not re.match(r"^[a-z0-9-\.]*$", hostname, re.IGNORECASE):return False- ps = PublicSuffixList(tldList)- sfx = ps.get_public_suffix(hostname, strict=True)+ ps = PublicSuffixList(tldList, only_icann=True, accept_unknown=False)+ sfx = ps.privatesuffix(hostname)return sfx != Nonedef isDomain(self, hostname, tldList):"""Check if the provided hostname string is a valid domain name.Given a possible hostname, check if it's a domain name- By checking whether it rests atop a TLD.+ By checking whether it rests atop a valid TLD.e.g. www.example.com = False because tld of hostname is com,and www.example has a . in it.@@ -1114,10 +1115,10 @@ class SpiderFoot:if not hostname:return False- ps = PublicSuffixList(tldList)- suffix = ps.get_public_suffix(hostname)- return hostname == suffix-+ ps = PublicSuffixList(tldList, only_icann=True, accept_unknown=False)+ sfx = ps.privatesuffix(hostname)+ return sfx == hostname+def validIP(self, address):"""Check if the provided string is a valid IPv4 address.@@ -1972,10 +1973,10 @@ class SpiderFoot:"""Remove key= and others from URLs to avoid credentials in logs."""pats = {- "key=\S+": "key=XXX",- "pass=\S+": "pass=XXX",- "user=\S+": "user=XXX",- "password=\S+": "password=XXX"+ r'key=\S+': "key=XXX",+ r'pass=\S+': "pass=XXX",+ r'user=\S+': "user=XXX",+ r'password=\S+': "password=XXX"}ret = url@@ -2728,146 +2729,3 @@ class SpiderFootEvent(object):def setSourceEventHash(self, srcHash):self.sourceEventHash = srcHash--"""-Public Suffix List module for Python.-See LICENSE.tp for applicable license.-"""---class PublicSuffixList(object):- def __init__(self, input_data):- """Reads and parses public suffix list.-- input_file is a file object or another iterable that returns- lines of a public suffix list file. If input_file is None, an- UTF-8 encoded file named "publicsuffix.txt" in the same- directory as this Python module is used.-- The file format is described at http://publicsuffix.org/list/- """-- #if input_file is None:- #input_path = os.path.join(os.path.dirname(__file__), 'publicsuffix.txt')- #input_file = codecs.open(input_path, "r", "utf-8")-- root = self._build_structure(input_data)- self.root = self._simplify(root)-- def _find_node(self, parent, parts):- if not parts:- return parent-- if len(parent) == 1:- parent.append({})-- if len(parent) != 2:- return None-- negate, children = parent-- child = parts.pop()-- child_node = children.get(child, None)-- if not child_node:- children[child] = child_node = [0]-- return self._find_node(child_node, parts)-- def _add_rule(self, root, rule):- if rule.startswith('!'):- negate = 1- rule = rule[1:]- else:- negate = 0-- parts = rule.split('.')- self._find_node(root, parts)[0] = negate-- def _simplify(self, node):- if len(node) == 1:- return node[0]-- return (node[0], dict((k, self._simplify(v)) for (k, v) in list(node[1].items())))-- def _build_structure(self, fp):- root = [0]-- for line in fp:- line = str(line).strip()- if line.startswith('//') or not line:- continue-- self._add_rule(root, line.split()[0].lstrip('.'))-- return root-- def _lookup_node(self, matches, depth, parent, parts):- if parent in (0, 1):- negate = parent- children = None- else:- negate, children = parent-- matches[-depth] = negate-- if depth < len(parts) and children:- for name in ('*', parts[-depth]):- child = children.get(name, None)- if child is not None:- self._lookup_node(matches, depth + 1, child, parts)-- def get_public_suffix(self, domain, strict=False):- """get_public_suffix("www.example.com") -> "example.com"-- Calling this function with a DNS name will return the- public suffix for that name.-- Note that for internationalized domains the list at- http://publicsuffix.org uses decoded names, so it is- up to the caller to decode any Punycode-encoded names.- """-- parts = domain.lower().lstrip('.').split('.')- hits = [None] * len(parts)-- if strict and parts[-1] not in self.root[1]:- return None-- self._lookup_node(hits, 1, self.root, parts)-- for i, what in enumerate(hits):- if what is not None and what == 0:- return '.'.join(parts[i:])---# Class for tracking the status of all running scans. Thread safe.-class SpiderFootScanStatus:- statusTable = dict()- lock = threading.Lock()-- def setStatus(self, scanId, status):- """Set scan status."""-- with self.lock:- self.statusTable[scanId] = status-- def getStatus(self, scanId):- """Get scan status."""-- with self.lock:- if scanId in self.statusTable:- return self.statusTable[scanId]- return None-- def getStatusAll(self):- """Set scan status for all scans."""-- with self.lock:- return self.statusTable--# Global variable accessed by various places to get the status of-# running scans.-globalScanStatus = SpiderFootScanStatus()-diff --git a/sfwebui.py b/sfwebui.pyindex 22150fb6..a09c949f 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -857,7 +857,6 @@ class SpiderFootWebUi:# Stop a scan (id variable is unnecessary for now given that only one simultaneous# scan is permitted.)def stopscanmulti(self, ids):- #Removing All globalScanStatus Refsdbh = SpiderFootDb(self.config)error = list()diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex bb5e082f..a9baaca6 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -33,6 +33,8 @@ class TestSpiderFoot(unittest.TestCase):'_torctlport': 9051,'__logstdout': False}++ test_tlds = "// ===BEGIN ICANN DOMAINS===\n\ncom\nnet\norg\n\n// // ===END ICANN DOMAINS===\n"def test_init_no_options(self):"""@@ -413,16 +415,18 @@ class TestSpiderFoot(unittest.TestCase):Test domainKeyword(self, domain, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tldskeyword = sf.domainKeyword('www.spiderfoot.net', sf.opts.get('_internettlds'))self.assertIsInstance(keyword, str)- self.assertEqual('wwwspiderfootnet', keyword)+ self.assertEqual('spiderfoot', keyword)def test_domain_keyword_invalid_domain_should_return_none(self):"""Test domainKeyword(self, domain, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tldskeyword = sf.domainKeyword("", sf.opts.get('_internettlds'))self.assertEqual(None, keyword)@@ -436,18 +440,20 @@ class TestSpiderFoot(unittest.TestCase):Test domainKeywords(self, domainList, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tldsdomain_list = ['www.example.com', 'localhost.local']keywords = sf.domainKeywords(domain_list, sf.opts.get('_internettlds'))self.assertIsInstance(keywords, set)- self.assertIn('localhostlocal', keywords)- self.assertIn('wwwexamplecom', keywords)+ self.assertIn('localhost', keywords)+ self.assertIn('example', keywords)def test_domain_keywords_invalid_domainlist_should_return_a_set(self):"""Test domainKeyword(self, domain, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tldskeywords = sf.domainKeywords("", sf.opts.get('_internettlds'))self.assertIsInstance(keywords, set)@@ -461,6 +467,7 @@ class TestSpiderFoot(unittest.TestCase):Test hostDomain(self, hostname, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tldshost_domain = sf.hostDomain(None, sf.opts.get('_internettlds'))self.assertEqual(None, host_domain)@@ -470,18 +477,19 @@ class TestSpiderFoot(unittest.TestCase):Test hostDomain(self, hostname, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tlds- host_domain = sf.hostDomain('local', sf.opts.get('_internettlds'))+ host_domain = sf.hostDomain('www.spiderfoot.net', sf.opts.get('_internettlds'))self.assertIsInstance(host_domain, str)- self.assertEqual('local', host_domain)+ self.assertEqual('spiderfoot.net', host_domain)host_domain = sf.hostDomain('spiderfoot.net', sf.opts.get('_internettlds'))self.assertIsInstance(host_domain, str)- self.assertEqual('net', host_domain)+ self.assertEqual('spiderfoot.net', host_domain)- host_domain = sf.hostDomain('www.spiderfoot.net', sf.opts.get('_internettlds'))+ host_domain = sf.hostDomain('abc.www.spiderfoot.net', sf.opts.get('_internettlds'))self.assertIsInstance(host_domain, str)- self.assertEqual('net', host_domain)+ self.assertEqual('spiderfoot.net', host_domain)def test_host_domain_invalid_tldlist_should_return_none(self):"""@@ -497,8 +505,9 @@ class TestSpiderFoot(unittest.TestCase):Test isDomain(self, hostname, tldList)"""sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tlds- is_domain = sf.isDomain('local', sf.opts.get('_internettlds'))+ is_domain = sf.isDomain('spiderfoot.net', sf.opts.get('_internettlds'))self.assertIsInstance(is_domain, bool)self.assertTrue(is_domain)@@ -512,6 +521,49 @@ class TestSpiderFoot(unittest.TestCase):self.assertIsInstance(is_domain, bool)self.assertFalse(is_domain)+ def test_is_domain_invalid_tld_should_return_false(self):+ """+ Test isDomain(self, hostname, tldList)+ """+ sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tlds++ is_domain = sf.isDomain('spiderfoot.not_a_tld', sf.opts.get('_internettlds'))+ self.assertIsInstance(is_domain, bool)+ self.assertFalse(is_domain)++ def test_valid_host_invalid_tldlist_should_return_false(self):+ """+ Test validHost(self, hostname, tldList)+ """+ sf = SpiderFoot(self.default_options)++ is_host = sf.validHost('spiderfoot.net', None)+ self.assertIsInstance(is_host, bool)+ self.assertFalse(is_host)++ def test_valid_host_valid_host_should_return_true(self):+ """+ Test validHost(self, hostname, tldList)+ """+ sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tlds++ is_host = sf.validHost('spiderfoot.net', sf.opts.get('_internettlds'))+ self.assertIsInstance(is_host, bool)+ self.assertTrue(is_host)++ def test_valid_host_invalid_host_should_return_false(self):+ """+ Test validHost(self, hostname, tldList)+ """+ sf = SpiderFoot(self.default_options)+ sf.opts['_internettlds'] = self.test_tlds++ is_valid = sf.validHost('something.gif', sf.opts.get('_internettlds'))+ self.assertIsInstance(is_valid, bool)+ self.assertFalse(is_valid)+def test_valid_ip_should_return_a_boolean(self):"""Test validIP(self, address)