SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
spiderfoot: Add SpiderFootHelpers (#1186)
bcoles114abfe5e3e4
diff --git a/sf.py b/sf.pyindex 29846211..375a3c81 100755--- a/sf.py+++ b/sf.py@@ -30,6 +30,7 @@ from cherrypy.lib import auth_digestfrom sflib import SpiderFootfrom sfscan import SpiderFootScannerfrom sfwebui import SpiderFootWebUi+from spiderfoot import SpiderFootHelpersfrom spiderfoot import SpiderFootDbfrom spiderfoot import __version__@@ -284,7 +285,7 @@ def start_scan(sfConfig, sfModules, args):target = f"\"{target}\""if "." not in target and not target.startswith("+") and '"' not in target:target = f"\"{target}\""- targetType = sf.targetType(target)+ targetType = SpiderFootHelpers.targetType(target)if not targetType:log.error(f"Could not determine target type. Invalid target: {target}")diff --git a/sflib.py b/sflib.pyindex 536b7640..6c8ca369 100644--- a/sflib.py+++ b/sflib.py@@ -39,8 +39,6 @@ import OpenSSLimport requestsimport urllib3from bs4 import BeautifulSoup, SoupStrainer-from networkx import nx-from networkx.readwrite.gexf import GEXFWriterfrom publicsuffixlist import PublicSuffixListfrom stem import Signalfrom stem.control import Controller@@ -207,190 +205,6 @@ class SpiderFoot:return val- def buildGraphData(self, data, flt=list()):- """Return a format-agnostic collection of tuples to use as the- basis for building graphs in various formats.-- Args:- data (str): TBD- flt (list): TBD-- Returns:- set: TBD- """- if not data:- return set()-- mapping = set()- entities = dict()- parents = dict()-- def get_next_parent_entities(item, pids):- ret = list()-- for [parent, entity_id] in parents[item]:- if entity_id in pids:- continue- if parent in entities:- ret.append(parent)- else:- pids.append(entity_id)- for p in get_next_parent_entities(parent, pids):- ret.append(p)- return ret-- for row in data:- if row[11] == "ENTITY" or row[11] == "INTERNAL":- # List of all valid entity values- if len(flt) > 0:- if row[4] in flt or row[11] == "INTERNAL":- entities[row[1]] = True- else:- entities[row[1]] = True-- if row[1] not in parents:- parents[row[1]] = list()- parents[row[1]].append([row[2], row[8]])-- for entity in entities:- for [parent, _id] in parents[entity]:- if parent in entities:- if entity != parent:- # self.log.debug(f"Adding entity parent: {parent}")- mapping.add((entity, parent))- else:- ppids = list()- # self.log.debug(f"Checking {parent} for entityship.")- next_parents = get_next_parent_entities(parent, ppids)- for next_parent in next_parents:- if entity != next_parent:- # self.log.debug("Adding next entity parent: {next_parent}")- mapping.add((entity, next_parent))- return mapping-- def buildGraphGexf(self, root, title, data, flt=[]):- """Convert supplied raw data into GEXF format (e.g. for Gephi)-- GEXF produced by PyGEXF doesn't work with SigmaJS because- SJS needs coordinates for each node.- flt is a list of event types to include, if not set everything is- included.-- Args:- root (str): TBD- title (str): unused- data (str): TBD- flt (list): TBD-- Returns:- str: TBD- """-- mapping = self.buildGraphData(data, flt)- graph = nx.Graph()-- nodelist = dict()- ncounter = 0- for pair in mapping:- (dst, src) = pair- col = ["0", "0", "0"]-- # Leave out this special case- if dst == "ROOT" or src == "ROOT":- continue-- if dst not in nodelist:- ncounter = ncounter + 1- if dst in root:- col = ["255", "0", "0"]- graph.node[dst]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}- nodelist[dst] = ncounter-- if src not in nodelist:- ncounter = ncounter + 1- if src in root:- col = ["255", "0", "0"]- graph.add_node(src)- graph.node[src]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}- nodelist[src] = ncounter-- graph.add_edge(src, dst)-- gexf = GEXFWriter(graph=graph)- return str(gexf).encode('utf-8')-- def buildGraphJson(self, root, data, flt=[]):- """Convert supplied raw data into JSON format for SigmaJS.-- Args:- root (str): TBD- data (str): TBD- flt (list): TBD-- Returns:- str: TBD- """-- mapping = self.buildGraphData(data, flt)- ret = dict()- ret['nodes'] = list()- ret['edges'] = list()-- nodelist = dict()- ecounter = 0- ncounter = 0- for pair in mapping:- (dst, src) = pair- col = "#000"-- # Leave out this special case- if dst == "ROOT" or src == "ROOT":- continue-- if dst not in nodelist:- ncounter = ncounter + 1-- if dst in root:- col = "#f00"-- ret['nodes'].append({- 'id': str(ncounter),- 'label': str(dst),- 'x': random.SystemRandom().randint(1, 1000),- 'y': random.SystemRandom().randint(1, 1000),- 'size': "1",- 'color': col- })-- nodelist[dst] = ncounter-- if src not in nodelist:- ncounter = ncounter + 1-- if src in root:- col = "#f00"-- ret['nodes'].append({- 'id': str(ncounter),- 'label': str(src),- 'x': random.SystemRandom().randint(1, 1000),- 'y': random.SystemRandom().randint(1, 1000),- 'size': "1",- 'color': col- })-- nodelist[src] = ncounter-- ecounter = ecounter + 1-- ret['edges'].append({- 'id': str(ecounter),- 'source': str(nodelist[src]),- 'target': str(nodelist[dst])- })-- return json.dumps(ret)-def genScanInstanceId(self):"""Generate an globally unique ID for this scan.@@ -793,40 +607,6 @@ class SpiderFoot:return returnOpts- def targetType(self, target):- """Return the scan target seed data type for the specified scan target input.-- Args:- target (str): scan target seed input-- Returns:- str: scan target seed data type- """- if not target:- return None-- # NOTE: the regex order is important- regexToType = [- {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$": "IP_ADDRESS"},- {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/\d+$": "NETBLOCK_OWNER"},- {r"^.*@.*$": "EMAILADDR"},- {r"^\+[0-9]+$": "PHONE_NUMBER"},- {r"^\".+\s+.+\"$": "HUMAN_NAME"},- {r"^\".+\"$": "USERNAME"},- {r"^[0-9]+$": "BGP_AS_OWNER"},- {r"^[0-9a-f:]+$": "IPV6_ADDRESS"},- {r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$": "INTERNET_NAME"},- {r"^([13][a-km-zA-HJ-NP-Z1-9]{25,34})$": "BITCOIN_ADDRESS"}- ]-- # Parse the target and set the target type- for rxpair in regexToType:- rx = list(rxpair.keys())[0]- if re.match(rx, target, re.IGNORECASE | re.UNICODE):- return list(rxpair.values())[0]-- return None-def modulesProducing(self, events):"""Return an array of modules that produce the list of types supplied.@@ -1418,57 +1198,6 @@ class SpiderFoot:return list(wd.keys())- def dataParentChildToTree(self, data):- """Converts a dictionary of k -> array to a nested- tree that can be digested by d3 for visualizations.-- Args:- data (dict): dictionary of k -> array-- Returns:- dict: nested tree- """-- if not isinstance(data, dict):- self.error("Data is not a dict")- return {}-- def get_children(needle, haystack):- ret = list()-- if needle not in list(haystack.keys()):- return None-- if haystack[needle] is None:- return None-- for c in haystack[needle]:- ret.append({"name": c, "children": get_children(c, haystack)})- return ret-- # Find the element with no parents, that's our root.- root = None- for k in list(data.keys()):- if data[k] is None:- continue-- contender = True- for ck in list(data.keys()):- if data[ck] is None:- continue-- if k in data[ck]:- contender = False-- if contender:- root = k- break-- if root is None:- return {}-- return {"name": root, "children": get_children(root, data)}-def resolveHost(self, host):"""Return a normalised resolution of a hostname.diff --git a/sfwebui.py b/sfwebui.pyindex c5ee1d2f..1469da86 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -27,6 +27,7 @@ from mako.template import Templatefrom secure import SecureHeadersfrom spiderfoot import SpiderFootDb+from spiderfoot import SpiderFootHelpersfrom spiderfoot import __version__from sflib import SpiderFootfrom sfscan import SpiderFootScanner@@ -348,7 +349,6 @@ class SpiderFootWebUi:# Export entities from scan results for visualisingdef scanviz(self, id, gexf="0"):dbh = SpiderFootDb(self.config)- sf = SpiderFoot(self.config)data = dbh.scanResultEvent(id, filterFp=True)scan = dbh.scanInstanceGet(id)root = scan[1]@@ -356,16 +356,15 @@ class SpiderFootWebUi:cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"cherrypy.response.headers['Content-Type'] = "application/gexf"cherrypy.response.headers['Pragma'] = "no-cache"- return sf.buildGraphGexf([root], "SpiderFoot Export", data)+ return SpiderFootHelpers.buildGraphGexf([root], "SpiderFoot Export", data)else:- return sf.buildGraphJson([root], data)+ return SpiderFootHelpers.buildGraphJson([root], data)scanviz.exposed = True# Export entities results from multiple scans in GEXF formatdef scanvizmulti(self, ids, gexf="1"):dbh = SpiderFootDb(self.config)- sf = SpiderFoot(self.config)data = list()roots = list()for id in ids.split(','):@@ -376,7 +375,7 @@ class SpiderFootWebUi:cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"cherrypy.response.headers['Content-Type'] = "application/gexf"cherrypy.response.headers['Pragma'] = "no-cache"- return sf.buildGraphGexf(roots, "SpiderFoot Export", data)+ return SpiderFootHelpers.buildGraphGexf(roots, "SpiderFoot Export", data)else:# Not implemented yetreturn None@@ -462,12 +461,12 @@ class SpiderFootWebUi:if "sfp__stor_stdout" in modlist:modlist.remove("sfp__stor_stdout")- targetType = sf.targetType(scantarget)+ targetType = SpiderFootHelpers.targetType(scantarget)if not targetType:# It must then be a name, as a re-run scan should always have a clean# target. Put quotes around the target value and try to determine the# target type again.- targetType = sf.targetType(f'"{scantarget}"')+ targetType = SpiderFootHelpers.targetType(f'"{scantarget}"')if targetType not in ["HUMAN_NAME", "BITCOIN_ADDRESS"]:scantarget = scantarget.lower()@@ -521,7 +520,7 @@ class SpiderFootWebUi:if "sfp__stor_stdout" in modlist:modlist.remove("sfp__stor_stdout")- targetType = sf.targetType(scantarget)+ targetType = SpiderFootHelpers.targetType(scantarget)if targetType is None:# Should never be triggered for a re-run scan..return self.error("Invalid target type. Could not recognize it as a target SpiderFoot supports.")@@ -573,7 +572,6 @@ class SpiderFootWebUi:None"""- sf = SpiderFoot(self.config)dbh = SpiderFootDb(self.config)types = dbh.eventTypes()info = dbh.scanInstanceGet(id)@@ -589,7 +587,7 @@ class SpiderFootWebUi:if scanname == "" or scantarget == "" or len(scanconfig) == 0:return self.error("Something went wrong internally.")- targetType = sf.targetType(scantarget)+ targetType = SpiderFootHelpers.targetType(scantarget)if targetType is None:# It must be a name, so wrap quotes around itscantarget = """ + scantarget + """@@ -1153,7 +1151,7 @@ class SpiderFootWebUi:modlist.append("sfp__stor_db")modlist.sort()- targetType = sf.targetType(scantarget)+ targetType = SpiderFootHelpers.targetType(scantarget)if targetType is None:if cherrypy.request.headers and 'application/json' in cherrypy.request.headers.get('Accept'):cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"@@ -1566,7 +1564,6 @@ class SpiderFootWebUi:cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"- sf = SpiderFoot(self.config)dbh = SpiderFootDb(self.config)pc = dict()datamap = dict()@@ -1578,7 +1575,7 @@ class SpiderFootWebUi:# Delete the ROOT key as it adds no value from a viz perspectivedel pc['ROOT']retdata = dict()- retdata['tree'] = sf.dataParentChildToTree(pc)+ retdata['tree'] = SpiderFootHelpers.dataParentChildToTree(pc)retdata['data'] = datamapreturn json.dumps(retdata).encode('utf-8')diff --git a/spiderfoot/__init__.py b/spiderfoot/__init__.pyindex b8a0b3d6..5b855c8c 100644--- a/spiderfoot/__init__.py+++ b/spiderfoot/__init__.py@@ -2,4 +2,5 @@ from .db import SpiderFootDbfrom .event import SpiderFootEventfrom .plugin import SpiderFootPluginfrom .target import SpiderFootTarget+from .helpers import SpiderFootHelpersfrom spiderfoot.__version__ import __version__diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.pynew file mode 100644index 00000000..42609655--- /dev/null+++ b/spiderfoot/helpers.py@@ -0,0 +1,282 @@+import json+import random+import re+from networkx import nx+from networkx.readwrite.gexf import GEXFWriter+++class SpiderFootHelpers():+ """SpiderFoot helper functions."""++ def targetType(target):+ """Return the scan target seed data type for the specified scan target input.++ Args:+ target (str): scan target seed input++ Returns:+ str: scan target seed data type+ """+ if not target:+ return None++ # NOTE: the regex order is important+ regexToType = [+ {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$": "IP_ADDRESS"},+ {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/\d+$": "NETBLOCK_OWNER"},+ {r"^.*@.*$": "EMAILADDR"},+ {r"^\+[0-9]+$": "PHONE_NUMBER"},+ {r"^\".+\s+.+\"$": "HUMAN_NAME"},+ {r"^\".+\"$": "USERNAME"},+ {r"^[0-9]+$": "BGP_AS_OWNER"},+ {r"^[0-9a-f:]+$": "IPV6_ADDRESS"},+ {r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$": "INTERNET_NAME"},+ {r"^([13][a-km-zA-HJ-NP-Z1-9]{25,34})$": "BITCOIN_ADDRESS"}+ ]++ # Parse the target and set the target type+ for rxpair in regexToType:+ rx = list(rxpair.keys())[0]+ if re.match(rx, target, re.IGNORECASE | re.UNICODE):+ return list(rxpair.values())[0]++ return None++ def buildGraphGexf(root, title, data, flt=[]):+ """Convert supplied raw data into GEXF format (e.g. for Gephi)++ GEXF produced by PyGEXF doesn't work with SigmaJS because+ SJS needs coordinates for each node.+ flt is a list of event types to include, if not set everything is+ included.++ Args:+ root (str): TBD+ title (str): unused+ data (list): scan result as list+ flt (list): TBD++ Returns:+ str: TBD+ """++ mapping = SpiderFootHelpers.buildGraphData(data, flt)+ graph = nx.Graph()++ nodelist = dict()+ ncounter = 0+ for pair in mapping:+ (dst, src) = pair+ col = ["0", "0", "0"]++ # Leave out this special case+ if dst == "ROOT" or src == "ROOT":+ continue++ if dst not in nodelist:+ ncounter = ncounter + 1+ if dst in root:+ col = ["255", "0", "0"]+ graph.node[dst]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}+ nodelist[dst] = ncounter++ if src not in nodelist:+ ncounter = ncounter + 1+ if src in root:+ col = ["255", "0", "0"]+ graph.add_node(src)+ graph.node[src]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}+ nodelist[src] = ncounter++ graph.add_edge(src, dst)++ gexf = GEXFWriter(graph=graph)+ return str(gexf).encode('utf-8')++ def buildGraphJson(root, data, flt=[]):+ """Convert supplied raw data into JSON format for SigmaJS.++ Args:+ root (str): TBD+ data (list): scan result as list+ flt (list): TBD++ Returns:+ str: TBD+ """++ mapping = SpiderFootHelpers.buildGraphData(data, flt)+ ret = dict()+ ret['nodes'] = list()+ ret['edges'] = list()++ nodelist = dict()+ ecounter = 0+ ncounter = 0+ for pair in mapping:+ (dst, src) = pair+ col = "#000"++ # Leave out this special case+ if dst == "ROOT" or src == "ROOT":+ continue++ if dst not in nodelist:+ ncounter = ncounter + 1++ if dst in root:+ col = "#f00"++ ret['nodes'].append({+ 'id': str(ncounter),+ 'label': str(dst),+ 'x': random.SystemRandom().randint(1, 1000),+ 'y': random.SystemRandom().randint(1, 1000),+ 'size': "1",+ 'color': col+ })++ nodelist[dst] = ncounter++ if src not in nodelist:+ ncounter = ncounter + 1++ if src in root:+ col = "#f00"++ ret['nodes'].append({+ 'id': str(ncounter),+ 'label': str(src),+ 'x': random.SystemRandom().randint(1, 1000),+ 'y': random.SystemRandom().randint(1, 1000),+ 'size': "1",+ 'color': col+ })++ nodelist[src] = ncounter++ ecounter = ecounter + 1++ ret['edges'].append({+ 'id': str(ecounter),+ 'source': str(nodelist[src]),+ 'target': str(nodelist[dst])+ })++ return json.dumps(ret)++ def buildGraphData(data, flt=list()):+ """Return a format-agnostic collection of tuples to use as the+ basis for building graphs in various formats.++ Args:+ data (list): Scan result as list+ flt (list): TBD++ Returns:+ set: TBD+ """+ if not data:+ return set()++ mapping = set()+ entities = dict()+ parents = dict()++ def get_next_parent_entities(item, pids):+ ret = list()++ for [parent, entity_id] in parents[item]:+ if entity_id in pids:+ continue+ if parent in entities:+ ret.append(parent)+ else:+ pids.append(entity_id)+ for p in get_next_parent_entities(parent, pids):+ ret.append(p)+ return ret++ for row in data:+ if len(row) != 15:+ # TODO: print error+ continue++ if row[11] == "ENTITY" or row[11] == "INTERNAL":+ # List of all valid entity values+ if len(flt) > 0:+ if row[4] in flt or row[11] == "INTERNAL":+ entities[row[1]] = True+ else:+ entities[row[1]] = True++ if row[1] not in parents:+ parents[row[1]] = list()+ parents[row[1]].append([row[2], row[8]])++ for entity in entities:+ for [parent, _id] in parents[entity]:+ if parent in entities:+ if entity != parent:+ # Add entity parent+ mapping.add((entity, parent))+ else:+ ppids = list()+ # Check parent for entityship.+ next_parents = get_next_parent_entities(parent, ppids)+ for next_parent in next_parents:+ if entity != next_parent:+ # Add next entity parent+ mapping.add((entity, next_parent))+ return mapping++ def dataParentChildToTree(data):+ """Converts a dictionary of k -> array to a nested+ tree that can be digested by d3 for visualizations.++ Args:+ data (dict): dictionary of k -> array++ Returns:+ dict: nested tree+ """++ if not isinstance(data, dict):+ # TODO: print error+ return {}++ def get_children(needle, haystack):+ ret = list()++ if needle not in list(haystack.keys()):+ return None++ if haystack[needle] is None:+ return None++ for c in haystack[needle]:+ ret.append({"name": c, "children": get_children(c, haystack)})+ return ret++ # Find the element with no parents, that's our root.+ root = None+ for k in list(data.keys()):+ if data[k] is None:+ continue++ contender = True+ for ck in list(data.keys()):+ if data[ck] is None:+ continue++ if k in data[ck]:+ contender = False++ if contender:+ root = k+ break++ if root is None:+ return {}++ return {"name": root, "children": get_children(root, data)}diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 9e96c082..c73f5a0b 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -106,48 +106,6 @@ class TestSpiderFoot(unittest.TestCase):opt_data = sf.optValueToData(invalid_type)self.assertEqual(opt_data, None)- def test_buildGraphData_should_return_a_set(self):- """- Test buildGraphData(self, data, flt=list())- """- sf = SpiderFoot(dict())-- graph_data = sf.buildGraphData('', '')- self.assertIsInstance(graph_data, set)-- graph_data = sf.buildGraphData(None, None)- self.assertIsInstance(graph_data, set)-- self.assertEqual('TBD', 'TBD')-- def test_buildGraphGexf_should_return_bytes(self):- """- Test buildGraphGexf(self, root, title, data, flt=[])- """- sf = SpiderFoot(dict())-- gexf = sf.buildGraphGexf(None, None, None)- self.assertIsInstance(gexf, bytes)-- gexf = sf.buildGraphGexf('', '', '')- self.assertIsInstance(gexf, bytes)-- self.assertEqual('TBD', 'TBD')-- def test_build_graph_json_should_return_a_string(self):- """- Test buildGraphJson(self, root, data, flt=list())- """- sf = SpiderFoot(dict())-- json = sf.buildGraphJson(None, None)- self.assertIsInstance(json, str)-- json = sf.buildGraphJson('', '')- self.assertIsInstance(json, str)-- self.assertEqual('TBD', 'TBD')-def test_genScanInstanceId_should_return_a_string(self):"""Test genScanInstanceId(self)@@ -329,48 +287,6 @@ class TestSpiderFoot(unittest.TestCase):cache_get = sf.cacheGet('', None)self.assertEqual(None, cache_get)- def test_target_type(self):- """- Test targetType(self, target)- """- sf = SpiderFoot(dict())-- target_type = sf.targetType("0.0.0.0")- self.assertEqual('IP_ADDRESS', target_type)- target_type = sf.targetType("[email protected]")- self.assertEqual('EMAILADDR', target_type)- target_type = sf.targetType("0.0.0.0/0")- self.assertEqual('NETBLOCK_OWNER', target_type)- target_type = sf.targetType("+1234567890")- self.assertEqual('PHONE_NUMBER', target_type)- target_type = sf.targetType('"Human Name"')- self.assertEqual('HUMAN_NAME', target_type)- target_type = sf.targetType('"abc123"')- self.assertEqual('USERNAME', target_type)- target_type = sf.targetType("1234567890")- self.assertEqual('BGP_AS_OWNER', target_type)- target_type = sf.targetType("::1")- self.assertEqual('IPV6_ADDRESS', target_type)- target_type = sf.targetType("spiderfoot.net")- self.assertEqual('INTERNET_NAME', target_type)- target_type = sf.targetType("1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R")- self.assertEqual('BITCOIN_ADDRESS', target_type)-- def test_target_type_invalid_seed_should_return_none(self):- """- Test targetType(self, target)- """- sf = SpiderFoot(dict())-- target_type = sf.targetType(None)- self.assertEqual(None, target_type)-- target_type = sf.targetType("")- self.assertEqual(None, target_type)-- target_type = sf.targetType('""')- self.assertEqual(None, target_type)-def test_modulesProducing_argument_events_should_return_a_list(self):"""Test modulesProducing(self, events)@@ -882,21 +798,6 @@ class TestSpiderFoot(unittest.TestCase):dict_names = sf.dictnames()self.assertIsInstance(dict_names, list)- def test_dataParentChildToTree_should_return_dict(self):- """- Test dataParentChildToTree(self, data)- """- sf = SpiderFoot(self.default_options)-- invalid_types = [None, "", list(), int()]- for invalid_type in invalid_types:- with self.subTest(invalid_type=invalid_type):- tree = sf.dataParentChildToTree(invalid_type)- self.assertIsInstance(tree, dict)-- tree = sf.dataParentChildToTree(dict())- self.assertIsInstance(tree, dict)-def test_resolve_host_should_return_list(self):"""Test resolveHost(self, host)diff --git a/test/unit/test_spiderfoothelpers.py b/test/unit/test_spiderfoothelpers.pynew file mode 100644index 00000000..8d25399e--- /dev/null+++ b/test/unit/test_spiderfoothelpers.py@@ -0,0 +1,123 @@+# test_spiderfoot.py+import pytest+import unittest++from spiderfoot import SpiderFootHelpers+++class TestSpiderFootHelpers(unittest.TestCase):+ """+ Test SpiderFootHelpers+ """++ def test_target_type(self):+ """+ Test targetType(target)+ """+ target_type = SpiderFootHelpers.targetType("0.0.0.0")+ self.assertEqual('IP_ADDRESS', target_type)+ target_type = SpiderFootHelpers.targetType("[email protected]")+ self.assertEqual('EMAILADDR', target_type)+ target_type = SpiderFootHelpers.targetType("0.0.0.0/0")+ self.assertEqual('NETBLOCK_OWNER', target_type)+ target_type = SpiderFootHelpers.targetType("+1234567890")+ self.assertEqual('PHONE_NUMBER', target_type)+ target_type = SpiderFootHelpers.targetType('"Human Name"')+ self.assertEqual('HUMAN_NAME', target_type)+ target_type = SpiderFootHelpers.targetType('"abc123"')+ self.assertEqual('USERNAME', target_type)+ target_type = SpiderFootHelpers.targetType("1234567890")+ self.assertEqual('BGP_AS_OWNER', target_type)+ target_type = SpiderFootHelpers.targetType("::1")+ self.assertEqual('IPV6_ADDRESS', target_type)+ target_type = SpiderFootHelpers.targetType("spiderfoot.net")+ self.assertEqual('INTERNET_NAME', target_type)+ target_type = SpiderFootHelpers.targetType("1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R")+ self.assertEqual('BITCOIN_ADDRESS', target_type)++ def test_target_type_invalid_seed_should_return_none(self):+ """+ Test targetType(target)+ """+ target_type = SpiderFootHelpers.targetType(None)+ self.assertEqual(None, target_type)++ target_type = SpiderFootHelpers.targetType("")+ self.assertEqual(None, target_type)++ target_type = SpiderFootHelpers.targetType('""')+ self.assertEqual(None, target_type)++ def test_buildGraphData_should_return_a_set(self):+ """+ Test buildGraphData(data, flt=list())+ """+ graph_data = SpiderFootHelpers.buildGraphData('', '')+ self.assertIsInstance(graph_data, set)++ graph_data = SpiderFootHelpers.buildGraphData(None, None)+ self.assertIsInstance(graph_data, set)++ graph_data = SpiderFootHelpers.buildGraphData(list(), list())+ self.assertIsInstance(graph_data, set)++ graph_data = SpiderFootHelpers.buildGraphData([])+ self.assertIsInstance(graph_data, set)++ graph_data = SpiderFootHelpers.buildGraphData(["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test"])+ self.assertIsInstance(graph_data, set)++ self.assertEqual('TBD', 'TBD')++ def test_buildGraphGexf_should_return_bytes(self):+ """+ Test buildGraphGexf(root, title, data, flt=[])+ """+ gexf = SpiderFootHelpers.buildGraphGexf(None, None, None)+ self.assertIsInstance(gexf, bytes)++ gexf = SpiderFootHelpers.buildGraphGexf('', '', '')+ self.assertIsInstance(gexf, bytes)++ gexf = SpiderFootHelpers.buildGraphGexf('test root', 'test title', [])+ self.assertIsInstance(gexf, bytes)++ gexf = SpiderFootHelpers.buildGraphGexf('test root', 'test title', [["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "ENTITY", "test", "test", "test"]])+ self.assertIsInstance(gexf, bytes)++ self.assertEqual('TBD', 'TBD')++ def test_buildGraphJson_should_return_a_string(self):+ """+ Test buildGraphJson(root, data, flt=list())+ """+ json = SpiderFootHelpers.buildGraphJson(None, None)+ self.assertIsInstance(json, str)++ json = SpiderFootHelpers.buildGraphJson('', '')+ self.assertIsInstance(json, str)++ json = SpiderFootHelpers.buildGraphJson('test root', [])+ self.assertIsInstance(json, str)++ json = SpiderFootHelpers.buildGraphJson('test root', [["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "ENTITY", "test", "test", "test"]])+ self.assertIsInstance(json, str)++ self.assertEqual('TBD', 'TBD')++ def test_dataParentChildToTree_should_return_dict(self):+ """+ Test dataParentChildToTree(data)+ """+ invalid_types = [None, "", list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ tree = SpiderFootHelpers.dataParentChildToTree(invalid_type)+ self.assertIsInstance(tree, dict)++ tree = SpiderFootHelpers.dataParentChildToTree(dict())+ self.assertIsInstance(tree, dict)++ tree = SpiderFootHelpers.dataParentChildToTree({"test": {"123": "456"}})+ self.assertIsInstance(tree, dict)