Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Tests: Add unit tests for SpiderFootCorrelator (#1672)

bcolese7decedabd79
diff --git a/spiderfoot/correlation.py b/spiderfoot/correlation.py
index 76d36938..1a1af631 100644
--- a/spiderfoot/correlation.py
+++ b/spiderfoot/correlation.py
@@ -61,6 +61,9 @@ class SpiderFootCorrelator:
if not isinstance(ruleset, dict):
raise TypeError(f"ruleset is {type(ruleset)}; expected dict()")
+ if not isinstance(dbh, SpiderFootDb):
+ raise TypeError(f"dbh is {type(dbh)}; expected SpiderFootDb()")
+
self.dbh = dbh
if scanId and not isinstance(scanId, str):
@@ -135,7 +138,12 @@ class SpiderFootCorrelator:
Returns:
dict: criteria to be used with SpiderFootDb.scanResultEvent()
+
+ Raises:
+ TypeError: argument type was invalid
"""
+ if not isinstance(matchrule, dict):
+ raise TypeError(f"matchrule is {type(matchrule)}; expected dict()")
criterias = dict()
@@ -212,7 +220,12 @@ class SpiderFootCorrelator:
Args:
events (dict): events
+
+ Raises:
+ TypeError: argument type was invalid
"""
+ if not isinstance(events, dict):
+ raise TypeError(f"events is {type(events)}; expected dict()")
event_chunks = [list(events.keys())[x:(x + 5000)] for x in range(0, len(list(events.keys())), 5000)]
@@ -234,7 +247,12 @@ class SpiderFootCorrelator:
Args:
events (dict): events
+
+ Raises:
+ TypeError: argument type was invalid
"""
+ if not isinstance(events, dict):
+ raise TypeError(f"events is {type(events)}; expected dict()")
event_chunks = [list(events.keys())[x:x + 5000] for x in range(0, len(list(events.keys())), 5000)]
@@ -257,7 +275,12 @@ class SpiderFootCorrelator:
Args:
events (dict): events
+
+ Raises:
+ TypeError: argument type was invalid
"""
+ if not isinstance(events, dict):
+ raise TypeError(f"events is {type(events)}; expected dict()")
entity_missing = dict()
for event_id in events:
@@ -830,7 +853,12 @@ class SpiderFootCorrelator:
Returns:
list: TBD
+
+ Raises:
+ TypeError: argument type was invalid
"""
+ if not isinstance(rule, dict):
+ raise TypeError(f"rule is {type(rule)}; expected dict()")
events = list()
buckets = dict()
@@ -880,10 +908,20 @@ class SpiderFootCorrelator:
Returns:
str: correlation rule title
+
+ Raises:
+ TypeError: argument type was invalid
"""
+ if not isinstance(rule, dict):
+ raise TypeError(f"rule is {type(rule)}; expected dict()")
+
+ if not isinstance(data, list):
+ raise TypeError(f"data is {type(data)}; expected list()")
+
title = rule['headline']
- if type(title) == dict:
+ if isinstance(title, dict):
title = title['text']
+
fields = re.findall(r"{([a-z\.]+)}", title)
for m in fields:
try:
@@ -938,6 +976,9 @@ class SpiderFootCorrelator:
Returns:
bool: correlation rule set is valid
"""
+ if not isinstance(rules, list):
+ return False
+
ok = True
for rule in rules:
if not self.check_rule_validity(rule):
@@ -956,9 +997,21 @@ class SpiderFootCorrelator:
Returns:
bool: correlation rule is valid
"""
- ok = True
+ if not isinstance(rule, dict):
+ return False
+
fields = set(rule.keys())
+ if not fields:
+ self.log.error("Rule is empty.")
+ return False
+
+ if not rule.get('id'):
+ self.log.error("Rule has no ID.")
+ return False
+
+ ok = True
+
for f in self.mandatory_components:
if f not in fields:
self.log.error(f"Mandatory rule component, {f}, not found in {rule['id']}.")
@@ -1009,8 +1062,13 @@ class SpiderFootCorrelator:
ok = False
continue
- if not rule[field].get(opt):
- self.log.error(f"Required field for {field} missing in {rule['id']}: {opt}")
+ elif type(rule[field]) == dict:
+ if not rule[field].get(opt):
+ self.log.error(f"Required field for {field} missing in {rule['id']}: {opt}")
+ ok = False
+
+ else:
+ self.log.error(f"Rule field '{field}' is not a list() or dict()")
ok = False
# Check if any of the options aren't valid
diff --git a/test/unit/spiderfoot/test_spiderfootcorrelator.py b/test/unit/spiderfoot/test_spiderfootcorrelator.py
new file mode 100644
index 00000000..3521c4d8
--- /dev/null
+++ b/test/unit/spiderfoot/test_spiderfootcorrelator.py
@@ -0,0 +1,174 @@
+# test_spiderfootcorrelator.py
+import unittest
+
+from spiderfoot import SpiderFootCorrelator, SpiderFootDb
+
+
+class TestSpiderFootCorrelator(unittest.TestCase):
+ """
+ Test SpiderFootCorrelator
+ """
+
+ def test_init_argument_dbh_invalid_type_should_raise_TypeError(self):
+ invalid_types = [None, str(), list(), dict(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ SpiderFootCorrelator(invalid_type, {})
+
+ def test_init_argument_ruleset_invalid_type_should_raise_TypeError(self):
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ SpiderFootCorrelator(None, invalid_type)
+
+ def test_init_argument_ruleset_invalid_rule_should_raise_SyntaxError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+
+ ruleset = {"sample rule": "invalid yaml"}
+ with self.assertRaises(SyntaxError):
+ SpiderFootCorrelator(sfdb, ruleset)
+
+ def test_run_correlations_invalid_scan_instance_should_raise_ValueError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+
+ correlator = SpiderFootCorrelator(sfdb, {}, 'example scan id')
+ with self.assertRaises(ValueError):
+ correlator.run_correlations()
+
+ def test_build_db_criteria_argument_matchrule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.build_db_criteria(invalid_type)
+
+ def test_enrich_event_sources_argument_rule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.enrich_event_sources(invalid_type)
+
+ def test_enrich_event_children_argument_rule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.enrich_event_children(invalid_type)
+
+ def test_enrich_event_entities_argument_rule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.enrich_event_entities(invalid_type)
+
+ def test_process_rule_argument_rule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.process_rule(invalid_type)
+
+ def test_build_correlation_title_argument_rule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.build_correlation_title(invalid_type, [])
+
+ def test_build_correlation_title_argument_data_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), dict(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.build_correlation_title({}, invalid_type)
+
+ def test_create_correlation_argument_rule_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.create_correlation(invalid_type, [], readonly=True)
+
+ def test_create_correlation_argument_data_invalid_type_should_raise_TypeError(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), dict(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ with self.assertRaises(TypeError):
+ correlator.create_correlation({}, invalid_type, readonly=True)
+
+ def test_check_ruleset_validity_should_return_bool(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ ruleset = [{"sample": "sample"}]
+ self.assertIsInstance(correlator.check_ruleset_validity(ruleset), bool)
+
+ invalid_types = [None, str(), list(), dict(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ self.assertIsInstance(correlator.check_ruleset_validity(invalid_type), bool)
+
+ def test_check_rule_validity_invalid_rule_should_return_false(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ invalid_types = [None, str(), list(), dict(), int()]
+ for invalid_type in invalid_types:
+ with self.subTest(invalid_type=invalid_type):
+ self.assertFalse(correlator.check_rule_validity(invalid_type))
+
+ def test_check_rule_validity_rule_missing_mandatory_field_should_return_false(self):
+ sfdb = SpiderFootDb(self.default_options, False)
+ correlator = SpiderFootCorrelator(sfdb, {})
+
+ rule = {
+ "id": "sample",
+ "collections": [],
+ "headline": "sample"
+ }
+ self.assertFalse(correlator.check_rule_validity(rule))
+
+ rule = {
+ "id": "sample",
+ "meta": "sample",
+ "headline": "sample"
+ }
+ self.assertFalse(correlator.check_rule_validity(rule))
+
+ rule = {
+ "id": "sample",
+ "meta": "sample",
+ "collections": []
+ }
+ self.assertFalse(correlator.check_rule_validity(rule))