Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

spiderfoot: Add SpiderFootHelpers (#1186)

bcoles114abfe5e3e4
diff --git a/sf.py b/sf.py
index 29846211..375a3c81 100755
--- a/sf.py
+++ b/sf.py
@@ -30,6 +30,7 @@ from cherrypy.lib import auth_digest
from sflib import SpiderFoot
from sfscan import SpiderFootScanner
from sfwebui import SpiderFootWebUi
+from spiderfoot import SpiderFootHelpers
from spiderfoot import SpiderFootDb
from spiderfoot import __version__
@@ -284,7 +285,7 @@ def start_scan(sfConfig, sfModules, args):
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
- targetType = sf.targetType(target)
+ targetType = SpiderFootHelpers.targetType(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
diff --git a/sflib.py b/sflib.py
index 536b7640..6c8ca369 100644
--- a/sflib.py
+++ b/sflib.py
@@ -39,8 +39,6 @@ import OpenSSL
import requests
import urllib3
from bs4 import BeautifulSoup, SoupStrainer
-from networkx import nx
-from networkx.readwrite.gexf import GEXFWriter
from publicsuffixlist import PublicSuffixList
from stem import Signal
from stem.control import Controller
@@ -207,190 +205,6 @@ class SpiderFoot:
return val
- def buildGraphData(self, data, flt=list()):
- """Return a format-agnostic collection of tuples to use as the
- basis for building graphs in various formats.
-
- Args:
- data (str): TBD
- flt (list): TBD
-
- Returns:
- set: TBD
- """
- if not data:
- return set()
-
- mapping = set()
- entities = dict()
- parents = dict()
-
- def get_next_parent_entities(item, pids):
- ret = list()
-
- for [parent, entity_id] in parents[item]:
- if entity_id in pids:
- continue
- if parent in entities:
- ret.append(parent)
- else:
- pids.append(entity_id)
- for p in get_next_parent_entities(parent, pids):
- ret.append(p)
- return ret
-
- for row in data:
- if row[11] == "ENTITY" or row[11] == "INTERNAL":
- # List of all valid entity values
- if len(flt) > 0:
- if row[4] in flt or row[11] == "INTERNAL":
- entities[row[1]] = True
- else:
- entities[row[1]] = True
-
- if row[1] not in parents:
- parents[row[1]] = list()
- parents[row[1]].append([row[2], row[8]])
-
- for entity in entities:
- for [parent, _id] in parents[entity]:
- if parent in entities:
- if entity != parent:
- # self.log.debug(f"Adding entity parent: {parent}")
- mapping.add((entity, parent))
- else:
- ppids = list()
- # self.log.debug(f"Checking {parent} for entityship.")
- next_parents = get_next_parent_entities(parent, ppids)
- for next_parent in next_parents:
- if entity != next_parent:
- # self.log.debug("Adding next entity parent: {next_parent}")
- mapping.add((entity, next_parent))
- return mapping
-
- def buildGraphGexf(self, root, title, data, flt=[]):
- """Convert supplied raw data into GEXF format (e.g. for Gephi)
-
- GEXF produced by PyGEXF doesn't work with SigmaJS because
- SJS needs coordinates for each node.
- flt is a list of event types to include, if not set everything is
- included.
-
- Args:
- root (str): TBD
- title (str): unused
- data (str): TBD
- flt (list): TBD
-
- Returns:
- str: TBD
- """
-
- mapping = self.buildGraphData(data, flt)
- graph = nx.Graph()
-
- nodelist = dict()
- ncounter = 0
- for pair in mapping:
- (dst, src) = pair
- col = ["0", "0", "0"]
-
- # Leave out this special case
- if dst == "ROOT" or src == "ROOT":
- continue
-
- if dst not in nodelist:
- ncounter = ncounter + 1
- if dst in root:
- col = ["255", "0", "0"]
- graph.node[dst]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}
- nodelist[dst] = ncounter
-
- if src not in nodelist:
- ncounter = ncounter + 1
- if src in root:
- col = ["255", "0", "0"]
- graph.add_node(src)
- graph.node[src]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}
- nodelist[src] = ncounter
-
- graph.add_edge(src, dst)
-
- gexf = GEXFWriter(graph=graph)
- return str(gexf).encode('utf-8')
-
- def buildGraphJson(self, root, data, flt=[]):
- """Convert supplied raw data into JSON format for SigmaJS.
-
- Args:
- root (str): TBD
- data (str): TBD
- flt (list): TBD
-
- Returns:
- str: TBD
- """
-
- mapping = self.buildGraphData(data, flt)
- ret = dict()
- ret['nodes'] = list()
- ret['edges'] = list()
-
- nodelist = dict()
- ecounter = 0
- ncounter = 0
- for pair in mapping:
- (dst, src) = pair
- col = "#000"
-
- # Leave out this special case
- if dst == "ROOT" or src == "ROOT":
- continue
-
- if dst not in nodelist:
- ncounter = ncounter + 1
-
- if dst in root:
- col = "#f00"
-
- ret['nodes'].append({
- 'id': str(ncounter),
- 'label': str(dst),
- 'x': random.SystemRandom().randint(1, 1000),
- 'y': random.SystemRandom().randint(1, 1000),
- 'size': "1",
- 'color': col
- })
-
- nodelist[dst] = ncounter
-
- if src not in nodelist:
- ncounter = ncounter + 1
-
- if src in root:
- col = "#f00"
-
- ret['nodes'].append({
- 'id': str(ncounter),
- 'label': str(src),
- 'x': random.SystemRandom().randint(1, 1000),
- 'y': random.SystemRandom().randint(1, 1000),
- 'size': "1",
- 'color': col
- })
-
- nodelist[src] = ncounter
-
- ecounter = ecounter + 1
-
- ret['edges'].append({
- 'id': str(ecounter),
- 'source': str(nodelist[src]),
- 'target': str(nodelist[dst])
- })
-
- return json.dumps(ret)
-
def genScanInstanceId(self):
"""Generate an globally unique ID for this scan.
@@ -793,40 +607,6 @@ class SpiderFoot:
return returnOpts
- def targetType(self, target):
- """Return the scan target seed data type for the specified scan target input.
-
- Args:
- target (str): scan target seed input
-
- Returns:
- str: scan target seed data type
- """
- if not target:
- return None
-
- # NOTE: the regex order is important
- regexToType = [
- {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$": "IP_ADDRESS"},
- {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/\d+$": "NETBLOCK_OWNER"},
- {r"^.*@.*$": "EMAILADDR"},
- {r"^\+[0-9]+$": "PHONE_NUMBER"},
- {r"^\".+\s+.+\"$": "HUMAN_NAME"},
- {r"^\".+\"$": "USERNAME"},
- {r"^[0-9]+$": "BGP_AS_OWNER"},
- {r"^[0-9a-f:]+$": "IPV6_ADDRESS"},
- {r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$": "INTERNET_NAME"},
- {r"^([13][a-km-zA-HJ-NP-Z1-9]{25,34})$": "BITCOIN_ADDRESS"}
- ]
-
- # Parse the target and set the target type
- for rxpair in regexToType:
- rx = list(rxpair.keys())[0]
- if re.match(rx, target, re.IGNORECASE | re.UNICODE):
- return list(rxpair.values())[0]
-
- return None
-
def modulesProducing(self, events):
"""Return an array of modules that produce the list of types supplied.
@@ -1418,57 +1198,6 @@ class SpiderFoot:
return list(wd.keys())
- def dataParentChildToTree(self, data):
- """Converts a dictionary of k -> array to a nested
- tree that can be digested by d3 for visualizations.
-
- Args:
- data (dict): dictionary of k -> array
-
- Returns:
- dict: nested tree
- """
-
- if not isinstance(data, dict):
- self.error("Data is not a dict")
- return {}
-
- def get_children(needle, haystack):
- ret = list()
-
- if needle not in list(haystack.keys()):
- return None
-
- if haystack[needle] is None:
- return None
-
- for c in haystack[needle]:
- ret.append({"name": c, "children": get_children(c, haystack)})
- return ret
-
- # Find the element with no parents, that's our root.
- root = None
- for k in list(data.keys()):
- if data[k] is None:
- continue
-
- contender = True
- for ck in list(data.keys()):
- if data[ck] is None:
- continue
-
- if k in data[ck]:
- contender = False
-
- if contender:
- root = k
- break
-
- if root is None:
- return {}
-
- return {"name": root, "children": get_children(root, data)}
-
def resolveHost(self, host):
"""Return a normalised resolution of a hostname.
diff --git a/sfwebui.py b/sfwebui.py
index c5ee1d2f..1469da86 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -27,6 +27,7 @@ from mako.template import Template
from secure import SecureHeaders
from spiderfoot import SpiderFootDb
+from spiderfoot import SpiderFootHelpers
from spiderfoot import __version__
from sflib import SpiderFoot
from sfscan import SpiderFootScanner
@@ -348,7 +349,6 @@ class SpiderFootWebUi:
# Export entities from scan results for visualising
def scanviz(self, id, gexf="0"):
dbh = SpiderFootDb(self.config)
- sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
@@ -356,16 +356,15 @@ class SpiderFootWebUi:
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
- return sf.buildGraphGexf([root], "SpiderFoot Export", data)
+ return SpiderFootHelpers.buildGraphGexf([root], "SpiderFoot Export", data)
else:
- return sf.buildGraphJson([root], data)
+ return SpiderFootHelpers.buildGraphJson([root], data)
scanviz.exposed = True
# Export entities results from multiple scans in GEXF format
def scanvizmulti(self, ids, gexf="1"):
dbh = SpiderFootDb(self.config)
- sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
@@ -376,7 +375,7 @@ class SpiderFootWebUi:
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
- return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
+ return SpiderFootHelpers.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
@@ -462,12 +461,12 @@ class SpiderFootWebUi:
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
- targetType = sf.targetType(scantarget)
+ targetType = SpiderFootHelpers.targetType(scantarget)
if not targetType:
# It must then be a name, as a re-run scan should always have a clean
# target. Put quotes around the target value and try to determine the
# target type again.
- targetType = sf.targetType(f'"{scantarget}"')
+ targetType = SpiderFootHelpers.targetType(f'"{scantarget}"')
if targetType not in ["HUMAN_NAME", "BITCOIN_ADDRESS"]:
scantarget = scantarget.lower()
@@ -521,7 +520,7 @@ class SpiderFootWebUi:
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
- targetType = sf.targetType(scantarget)
+ targetType = SpiderFootHelpers.targetType(scantarget)
if targetType is None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as a target SpiderFoot supports.")
@@ -573,7 +572,6 @@ class SpiderFootWebUi:
None
"""
- sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
@@ -589,7 +587,7 @@ class SpiderFootWebUi:
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
- targetType = sf.targetType(scantarget)
+ targetType = SpiderFootHelpers.targetType(scantarget)
if targetType is None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
@@ -1153,7 +1151,7 @@ class SpiderFootWebUi:
modlist.append("sfp__stor_db")
modlist.sort()
- targetType = sf.targetType(scantarget)
+ targetType = SpiderFootHelpers.targetType(scantarget)
if targetType is None:
if cherrypy.request.headers and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
@@ -1566,7 +1564,6 @@ class SpiderFootWebUi:
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
- sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
@@ -1578,7 +1575,7 @@ class SpiderFootWebUi:
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
- retdata['tree'] = sf.dataParentChildToTree(pc)
+ retdata['tree'] = SpiderFootHelpers.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata).encode('utf-8')
diff --git a/spiderfoot/__init__.py b/spiderfoot/__init__.py
index b8a0b3d6..5b855c8c 100644
--- a/spiderfoot/__init__.py
+++ b/spiderfoot/__init__.py
@@ -2,4 +2,5 @@ from .db import SpiderFootDb
from .event import SpiderFootEvent
from .plugin import SpiderFootPlugin
from .target import SpiderFootTarget
+from .helpers import SpiderFootHelpers
from spiderfoot.__version__ import __version__
diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.py
new file mode 100644
index 00000000..42609655
--- /dev/null
+++ b/spiderfoot/helpers.py
@@ -0,0 +1,282 @@
+import json
+import random
+import re
+from networkx import nx
+from networkx.readwrite.gexf import GEXFWriter
+
+
+class SpiderFootHelpers():
+ """SpiderFoot helper functions."""
+
+ def targetType(target):
+ """Return the scan target seed data type for the specified scan target input.
+
+ Args:
+ target (str): scan target seed input
+
+ Returns:
+ str: scan target seed data type
+ """
+ if not target:
+ return None
+
+ # NOTE: the regex order is important
+ regexToType = [
+ {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$": "IP_ADDRESS"},
+ {r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/\d+$": "NETBLOCK_OWNER"},
+ {r"^.*@.*$": "EMAILADDR"},
+ {r"^\+[0-9]+$": "PHONE_NUMBER"},
+ {r"^\".+\s+.+\"$": "HUMAN_NAME"},
+ {r"^\".+\"$": "USERNAME"},
+ {r"^[0-9]+$": "BGP_AS_OWNER"},
+ {r"^[0-9a-f:]+$": "IPV6_ADDRESS"},
+ {r"^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$": "INTERNET_NAME"},
+ {r"^([13][a-km-zA-HJ-NP-Z1-9]{25,34})$": "BITCOIN_ADDRESS"}
+ ]
+
+ # Parse the target and set the target type
+ for rxpair in regexToType:
+ rx = list(rxpair.keys())[0]
+ if re.match(rx, target, re.IGNORECASE | re.UNICODE):
+ return list(rxpair.values())[0]
+
+ return None
+
+ def buildGraphGexf(root, title, data, flt=[]):
+ """Convert supplied raw data into GEXF format (e.g. for Gephi)
+
+ GEXF produced by PyGEXF doesn't work with SigmaJS because
+ SJS needs coordinates for each node.
+ flt is a list of event types to include, if not set everything is
+ included.
+
+ Args:
+ root (str): TBD
+ title (str): unused
+ data (list): scan result as list
+ flt (list): TBD
+
+ Returns:
+ str: TBD
+ """
+
+ mapping = SpiderFootHelpers.buildGraphData(data, flt)
+ graph = nx.Graph()
+
+ nodelist = dict()
+ ncounter = 0
+ for pair in mapping:
+ (dst, src) = pair
+ col = ["0", "0", "0"]
+
+ # Leave out this special case
+ if dst == "ROOT" or src == "ROOT":
+ continue
+
+ if dst not in nodelist:
+ ncounter = ncounter + 1
+ if dst in root:
+ col = ["255", "0", "0"]
+ graph.node[dst]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}
+ nodelist[dst] = ncounter
+
+ if src not in nodelist:
+ ncounter = ncounter + 1
+ if src in root:
+ col = ["255", "0", "0"]
+ graph.add_node(src)
+ graph.node[src]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}
+ nodelist[src] = ncounter
+
+ graph.add_edge(src, dst)
+
+ gexf = GEXFWriter(graph=graph)
+ return str(gexf).encode('utf-8')
+
+ def buildGraphJson(root, data, flt=[]):
+ """Convert supplied raw data into JSON format for SigmaJS.
+
+ Args:
+ root (str): TBD
+ data (list): scan result as list
+ flt (list): TBD
+
+ Returns:
+ str: TBD
+ """
+
+ mapping = SpiderFootHelpers.buildGraphData(data, flt)
+ ret = dict()
+ ret['nodes'] = list()
+ ret['edges'] = list()
+
+ nodelist = dict()
+ ecounter = 0
+ ncounter = 0
+ for pair in mapping:
+ (dst, src) = pair
+ col = "#000"
+
+ # Leave out this special case
+ if dst == "ROOT" or src == "ROOT":
+ continue
+
+ if dst not in nodelist:
+ ncounter = ncounter + 1
+
+ if dst in root:
+ col = "#f00"
+
+ ret['nodes'].append({
+ 'id': str(ncounter),
+ 'label': str(dst),
+ 'x': random.SystemRandom().randint(1, 1000),
+ 'y': random.SystemRandom().randint(1, 1000),
+ 'size': "1",
+ 'color': col
+ })
+
+ nodelist[dst] = ncounter
+
+ if src not in nodelist:
+ ncounter = ncounter + 1
+
+ if src in root:
+ col = "#f00"
+
+ ret['nodes'].append({
+ 'id': str(ncounter),
+ 'label': str(src),
+ 'x': random.SystemRandom().randint(1, 1000),
+ 'y': random.SystemRandom().randint(1, 1000),
+ 'size': "1",
+ 'color': col
+ })
+
+ nodelist[src] = ncounter
+
+ ecounter = ecounter + 1
+
+ ret['edges'].append({
+ 'id': str(ecounter),
+ 'source': str(nodelist[src]),
+ 'target': str(nodelist[dst])
+ })
+
+ return json.dumps(ret)
+
+ def buildGraphData(data, flt=list()):
+ """Return a format-agnostic collection of tuples to use as the
+ basis for building graphs in various formats.
+
+ Args:
+ data (list): Scan result as list
+ flt (list): TBD
+
+ Returns:
+ set: TBD
+ """
+ if not data:
+ return set()
+
+ mapping = set()
+ entities = dict()
+ parents = dict()
+
+ def get_next_parent_entities(item, pids):
+ ret = list()
+
+ for [parent, entity_id] in parents[item]:
+ if entity_id in pids:
+ continue
+ if parent in entities:
+ ret.append(parent)
+ else:
+ pids.append(entity_id)
+ for p in get_next_parent_entities(parent, pids):
+ ret.append(p)
+ return ret
+
+ for row in data:
+ if len(row) != 15:
+ # TODO: print error
+ continue
+
+ if row[11] == "ENTITY" or row[11] == "INTERNAL":
+ # List of all valid entity values
+ if len(flt) > 0:
+ if row[4] in flt or row[11] == "INTERNAL":
+ entities[row[1]] = True
+ else:
+ entities[row[1]] = True
+
+ if row[1] not in parents:
+ parents[row[1]] = list()
+ parents[row[1]].append([row[2], row[8]])
+
+ for entity in entities:
+ for [parent, _id] in parents[entity]:
+ if parent in entities:
+ if entity != parent:
+ # Add entity parent
+ mapping.add((entity, parent))
+ else:
+ ppids = list()
+ # Check parent for entityship.
+ next_parents = get_next_parent_entities(parent, ppids)
+ for next_parent in next_parents:
+ if entity != next_parent:
+ # Add next entity parent
+ mapping.add((entity, next_parent))
+ return mapping
+
+ def dataParentChildToTree(data):
+ """Converts a dictionary of k -> array to a nested
+ tree that can be digested by d3 for visualizations.
+
+ Args:
+ data (dict): dictionary of k -> array
+
+ Returns:
+ dict: nested tree
+ """
+
+ if not isinstance(data, dict):
+ # TODO: print error
+ return {}
+
+ def get_children(needle, haystack):
+ ret = list()
+
+ if needle not in list(haystack.keys()):
+ return None
+
+ if haystack[needle] is None:
+ return None
+
+ for c in haystack[needle]:
+ ret.append({"name": c, "children": get_children(c, haystack)})
+ return ret
+
+ # Find the element with no parents, that's our root.
+ root = None
+ for k in list(data.keys()):
+ if data[k] is None:
+ continue
+
+ contender = True
+ for ck in list(data.keys()):
+ if data[ck] is None:
+ continue
+
+ if k in data[ck]:
+ contender = False
+
+ if contender:
+ root = k
+ break
+
+ if root is None:
+ return {}
+
+ return {"name": root, "children": get_children(root, data)}
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index 9e96c082..c73f5a0b 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -106,48 +106,6 @@ class TestSpiderFoot(unittest.TestCase):
opt_data = sf.optValueToData(invalid_type)
self.assertEqual(opt_data, None)
- def test_buildGraphData_should_return_a_set(self):
- """
- Test buildGraphData(self, data, flt=list())
- """
- sf = SpiderFoot(dict())
-
- graph_data = sf.buildGraphData('', '')
- self.assertIsInstance(graph_data, set)
-
- graph_data = sf.buildGraphData(None, None)
- self.assertIsInstance(graph_data, set)
-
- self.assertEqual('TBD', 'TBD')
-
- def test_buildGraphGexf_should_return_bytes(self):
- """
- Test buildGraphGexf(self, root, title, data, flt=[])
- """
- sf = SpiderFoot(dict())
-
- gexf = sf.buildGraphGexf(None, None, None)
- self.assertIsInstance(gexf, bytes)
-
- gexf = sf.buildGraphGexf('', '', '')
- self.assertIsInstance(gexf, bytes)
-
- self.assertEqual('TBD', 'TBD')
-
- def test_build_graph_json_should_return_a_string(self):
- """
- Test buildGraphJson(self, root, data, flt=list())
- """
- sf = SpiderFoot(dict())
-
- json = sf.buildGraphJson(None, None)
- self.assertIsInstance(json, str)
-
- json = sf.buildGraphJson('', '')
- self.assertIsInstance(json, str)
-
- self.assertEqual('TBD', 'TBD')
-
def test_genScanInstanceId_should_return_a_string(self):
"""
Test genScanInstanceId(self)
@@ -329,48 +287,6 @@ class TestSpiderFoot(unittest.TestCase):
cache_get = sf.cacheGet('', None)
self.assertEqual(None, cache_get)
- def test_target_type(self):
- """
- Test targetType(self, target)
- """
- sf = SpiderFoot(dict())
-
- target_type = sf.targetType("0.0.0.0")
- self.assertEqual('IP_ADDRESS', target_type)
- target_type = sf.targetType("[email protected]")
- self.assertEqual('EMAILADDR', target_type)
- target_type = sf.targetType("0.0.0.0/0")
- self.assertEqual('NETBLOCK_OWNER', target_type)
- target_type = sf.targetType("+1234567890")
- self.assertEqual('PHONE_NUMBER', target_type)
- target_type = sf.targetType('"Human Name"')
- self.assertEqual('HUMAN_NAME', target_type)
- target_type = sf.targetType('"abc123"')
- self.assertEqual('USERNAME', target_type)
- target_type = sf.targetType("1234567890")
- self.assertEqual('BGP_AS_OWNER', target_type)
- target_type = sf.targetType("::1")
- self.assertEqual('IPV6_ADDRESS', target_type)
- target_type = sf.targetType("spiderfoot.net")
- self.assertEqual('INTERNET_NAME', target_type)
- target_type = sf.targetType("1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R")
- self.assertEqual('BITCOIN_ADDRESS', target_type)
-
- def test_target_type_invalid_seed_should_return_none(self):
- """
- Test targetType(self, target)
- """
- sf = SpiderFoot(dict())
-
- target_type = sf.targetType(None)
- self.assertEqual(None, target_type)
-
- target_type = sf.targetType("")
- self.assertEqual(None, target_type)
-
- target_type = sf.targetType('""')
- self.assertEqual(None, target_type)
-
def test_modulesProducing_argument_events_should_return_a_list(self):
"""
Test modulesProducing(self, events)
@@ -882,21 +798,6 @@ class TestSpiderFoot(unittest.TestCase):
dict_names = sf.dictnames()
self.assertIsInstance(dict_names, list)
- def test_dataParentChildToTree_should_return_dict(self):
- """
- Test dataParentChildToTree(self, data)
- """
- sf = SpiderFoot(self.default_options)
-
- invalid_types = [None, "", list(), int()]
- for invalid_type in invalid_types:
- with self.subTest(invalid_type=invalid_type):
- tree = sf.dataParentChildToTree(invalid_type)
- self.assertIsInstance(tree, dict)
-
- tree = sf.dataParentChildToTree(dict())
- self.assertIsInstance(tree, dict)
-
def test_resolve_host_should_return_list(self):
"""
Test resolveHost(self, host)
diff --git a/test/unit/test_spiderfoothelpers.py b/test/unit/test_spiderfoothelpers.py
new file mode 100644
index 00000000..8d25399e
--- /dev/null
+++ b/test/unit/test_spiderfoothelpers.py
@@ -0,0 +1,123 @@
+# test_spiderfoot.py
+import pytest
+import unittest
+
+from spiderfoot import SpiderFootHelpers
+
+
+class TestSpiderFootHelpers(unittest.TestCase):
+ """
+ Test SpiderFootHelpers
+ """
+
+ def test_target_type(self):
+ """
+ Test targetType(target)
+ """
+ target_type = SpiderFootHelpers.targetType("0.0.0.0")
+ self.assertEqual('IP_ADDRESS', target_type)
+ target_type = SpiderFootHelpers.targetType("[email protected]")
+ self.assertEqual('EMAILADDR', target_type)
+ target_type = SpiderFootHelpers.targetType("0.0.0.0/0")
+ self.assertEqual('NETBLOCK_OWNER', target_type)
+ target_type = SpiderFootHelpers.targetType("+1234567890")
+ self.assertEqual('PHONE_NUMBER', target_type)
+ target_type = SpiderFootHelpers.targetType('"Human Name"')
+ self.assertEqual('HUMAN_NAME', target_type)
+ target_type = SpiderFootHelpers.targetType('"abc123"')
+ self.assertEqual('USERNAME', target_type)
+ target_type = SpiderFootHelpers.targetType("1234567890")
+ self.assertEqual('BGP_AS_OWNER', target_type)
+ target_type = SpiderFootHelpers.targetType("::1")
+ self.assertEqual('IPV6_ADDRESS', target_type)
+ target_type = SpiderFootHelpers.targetType("spiderfoot.net")
+ self.assertEqual('INTERNET_NAME', target_type)
+ target_type = SpiderFootHelpers.targetType("1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R")
+ self.assertEqual('BITCOIN_ADDRESS', target_type)
+
+ def test_target_type_invalid_seed_should_return_none(self):
+ """
+ Test targetType(target)
+ """
+ target_type = SpiderFootHelpers.targetType(None)
+ self.assertEqual(None, target_type)
+
+ target_type = SpiderFootHelpers.targetType("")
+ self.assertEqual(None, target_type)
+
+ target_type = SpiderFootHelpers.targetType('""')
+ self.assertEqual(None, target_type)
+
+ def test_buildGraphData_should_return_a_set(self):
+ """
+ Test buildGraphData(data, flt=list())
+ """
+ graph_data = SpiderFootHelpers.buildGraphData('', '')
+ self.assertIsInstance(graph_data, set)
+
+ graph_data = SpiderFootHelpers.buildGraphData(None, None)
+ self.assertIsInstance(graph_data, set)
+
+ graph_data = SpiderFootHelpers.buildGraphData(list(), list())
+ self.assertIsInstance(graph_data, set)
+
+ graph_data = SpiderFootHelpers.buildGraphData([])
+ self.assertIsInstance(graph_data, set)
+
+ graph_data = SpiderFootHelpers.buildGraphData(["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test"])
+ self.assertIsInstance(graph_data, set)
+
+ self.assertEqual('TBD', 'TBD')
+
+ def test_buildGraphGexf_should_return_bytes(self):
+ """
+ Test buildGraphGexf(root, title, data, flt=[])
+ """
+ gexf = SpiderFootHelpers.buildGraphGexf(None, None, None)
+ self.assertIsInstance(gexf, bytes)
+
+ gexf = SpiderFootHelpers.buildGraphGexf('', '', '')
+ self.assertIsInstance(gexf, bytes)
+
+ gexf = SpiderFootHelpers.buildGraphGexf('test root', 'test title', [])
+ self.assertIsInstance(gexf, bytes)
+
+ gexf = SpiderFootHelpers.buildGraphGexf('test root', 'test title', [["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "ENTITY", "test", "test", "test"]])
+ self.assertIsInstance(gexf, bytes)
+
+ self.assertEqual('TBD', 'TBD')
+
+ def test_buildGraphJson_should_return_a_string(self):
+ """
+ Test buildGraphJson(root, data, flt=list())
+ """
+ json = SpiderFootHelpers.buildGraphJson(None, None)
+ self.assertIsInstance(json, str)
+
+ json = SpiderFootHelpers.buildGraphJson('', '')
+ self.assertIsInstance(json, str)
+
+ json = SpiderFootHelpers.buildGraphJson('test root', [])
+ self.assertIsInstance(json, str)
+
+ json = SpiderFootHelpers.buildGraphJson('test root', [["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "ENTITY", "test", "test", "test"]])
+ self.assertIsInstance(json, str)
+
+ self.assertEqual('TBD', 'TBD')
+
+ def test_dataParentChildToTree_should_return_dict(self):
+ """
+ Test dataParentChildToTree(data)
+ """
+ invalid_types = [None, "", list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ tree = SpiderFootHelpers.dataParentChildToTree(invalid_type)
+ self.assertIsInstance(tree, dict)
+
+ tree = SpiderFootHelpers.dataParentChildToTree(dict())
+ self.assertIsInstance(tree, dict)
+
+ tree = SpiderFootHelpers.dataParentChildToTree({"test": {"123": "456"}})
+ self.assertIsInstance(tree, dict)