SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Tests: Add unit tests for SpiderFootCorrelator (#1672)
bcolese7decedabd79
diff --git a/spiderfoot/correlation.py b/spiderfoot/correlation.pyindex 76d36938..1a1af631 100644--- a/spiderfoot/correlation.py+++ b/spiderfoot/correlation.py@@ -61,6 +61,9 @@ class SpiderFootCorrelator:if not isinstance(ruleset, dict):raise TypeError(f"ruleset is {type(ruleset)}; expected dict()")+ if not isinstance(dbh, SpiderFootDb):+ raise TypeError(f"dbh is {type(dbh)}; expected SpiderFootDb()")+self.dbh = dbhif scanId and not isinstance(scanId, str):@@ -135,7 +138,12 @@ class SpiderFootCorrelator:Returns:dict: criteria to be used with SpiderFootDb.scanResultEvent()++ Raises:+ TypeError: argument type was invalid"""+ if not isinstance(matchrule, dict):+ raise TypeError(f"matchrule is {type(matchrule)}; expected dict()")criterias = dict()@@ -212,7 +220,12 @@ class SpiderFootCorrelator:Args:events (dict): events++ Raises:+ TypeError: argument type was invalid"""+ if not isinstance(events, dict):+ raise TypeError(f"events is {type(events)}; expected dict()")event_chunks = [list(events.keys())[x:(x + 5000)] for x in range(0, len(list(events.keys())), 5000)]@@ -234,7 +247,12 @@ class SpiderFootCorrelator:Args:events (dict): events++ Raises:+ TypeError: argument type was invalid"""+ if not isinstance(events, dict):+ raise TypeError(f"events is {type(events)}; expected dict()")event_chunks = [list(events.keys())[x:x + 5000] for x in range(0, len(list(events.keys())), 5000)]@@ -257,7 +275,12 @@ class SpiderFootCorrelator:Args:events (dict): events++ Raises:+ TypeError: argument type was invalid"""+ if not isinstance(events, dict):+ raise TypeError(f"events is {type(events)}; expected dict()")entity_missing = dict()for event_id in events:@@ -830,7 +853,12 @@ class SpiderFootCorrelator:Returns:list: TBD++ Raises:+ TypeError: argument type was invalid"""+ if not isinstance(rule, dict):+ raise TypeError(f"rule is {type(rule)}; expected dict()")events = list()buckets = dict()@@ -880,10 +908,20 @@ class SpiderFootCorrelator:Returns:str: correlation rule title++ Raises:+ TypeError: argument type was invalid"""+ if not isinstance(rule, dict):+ raise TypeError(f"rule is {type(rule)}; expected dict()")++ if not isinstance(data, list):+ raise TypeError(f"data is {type(data)}; expected list()")+title = rule['headline']- if type(title) == dict:+ if isinstance(title, dict):title = title['text']+fields = re.findall(r"{([a-z\.]+)}", title)for m in fields:try:@@ -938,6 +976,9 @@ class SpiderFootCorrelator:Returns:bool: correlation rule set is valid"""+ if not isinstance(rules, list):+ return False+ok = Truefor rule in rules:if not self.check_rule_validity(rule):@@ -956,9 +997,21 @@ class SpiderFootCorrelator:Returns:bool: correlation rule is valid"""- ok = True+ if not isinstance(rule, dict):+ return False+fields = set(rule.keys())+ if not fields:+ self.log.error("Rule is empty.")+ return False++ if not rule.get('id'):+ self.log.error("Rule has no ID.")+ return False++ ok = True+for f in self.mandatory_components:if f not in fields:self.log.error(f"Mandatory rule component, {f}, not found in {rule['id']}.")@@ -1009,8 +1062,13 @@ class SpiderFootCorrelator:ok = Falsecontinue- if not rule[field].get(opt):- self.log.error(f"Required field for {field} missing in {rule['id']}: {opt}")+ elif type(rule[field]) == dict:+ if not rule[field].get(opt):+ self.log.error(f"Required field for {field} missing in {rule['id']}: {opt}")+ ok = False++ else:+ self.log.error(f"Rule field '{field}' is not a list() or dict()")ok = False# Check if any of the options aren't validdiff --git a/test/unit/spiderfoot/test_spiderfootcorrelator.py b/test/unit/spiderfoot/test_spiderfootcorrelator.pynew file mode 100644index 00000000..3521c4d8--- /dev/null+++ b/test/unit/spiderfoot/test_spiderfootcorrelator.py@@ -0,0 +1,174 @@+# test_spiderfootcorrelator.py+import unittest++from spiderfoot import SpiderFootCorrelator, SpiderFootDb+++class TestSpiderFootCorrelator(unittest.TestCase):+ """+ Test SpiderFootCorrelator+ """++ def test_init_argument_dbh_invalid_type_should_raise_TypeError(self):+ invalid_types = [None, str(), list(), dict(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ SpiderFootCorrelator(invalid_type, {})++ def test_init_argument_ruleset_invalid_type_should_raise_TypeError(self):+ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ SpiderFootCorrelator(None, invalid_type)++ def test_init_argument_ruleset_invalid_rule_should_raise_SyntaxError(self):+ sfdb = SpiderFootDb(self.default_options, False)++ ruleset = {"sample rule": "invalid yaml"}+ with self.assertRaises(SyntaxError):+ SpiderFootCorrelator(sfdb, ruleset)++ def test_run_correlations_invalid_scan_instance_should_raise_ValueError(self):+ sfdb = SpiderFootDb(self.default_options, False)++ correlator = SpiderFootCorrelator(sfdb, {}, 'example scan id')+ with self.assertRaises(ValueError):+ correlator.run_correlations()++ def test_build_db_criteria_argument_matchrule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.build_db_criteria(invalid_type)++ def test_enrich_event_sources_argument_rule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.enrich_event_sources(invalid_type)++ def test_enrich_event_children_argument_rule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.enrich_event_children(invalid_type)++ def test_enrich_event_entities_argument_rule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.enrich_event_entities(invalid_type)++ def test_process_rule_argument_rule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.process_rule(invalid_type)++ def test_build_correlation_title_argument_rule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.build_correlation_title(invalid_type, [])++ def test_build_correlation_title_argument_data_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), dict(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.build_correlation_title({}, invalid_type)++ def test_create_correlation_argument_rule_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.create_correlation(invalid_type, [], readonly=True)++ def test_create_correlation_argument_data_invalid_type_should_raise_TypeError(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), dict(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError):+ correlator.create_correlation({}, invalid_type, readonly=True)++ def test_check_ruleset_validity_should_return_bool(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ ruleset = [{"sample": "sample"}]+ self.assertIsInstance(correlator.check_ruleset_validity(ruleset), bool)++ invalid_types = [None, str(), list(), dict(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ self.assertIsInstance(correlator.check_ruleset_validity(invalid_type), bool)++ def test_check_rule_validity_invalid_rule_should_return_false(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ invalid_types = [None, str(), list(), dict(), int()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ self.assertFalse(correlator.check_rule_validity(invalid_type))++ def test_check_rule_validity_rule_missing_mandatory_field_should_return_false(self):+ sfdb = SpiderFootDb(self.default_options, False)+ correlator = SpiderFootCorrelator(sfdb, {})++ rule = {+ "id": "sample",+ "collections": [],+ "headline": "sample"+ }+ self.assertFalse(correlator.check_rule_validity(rule))++ rule = {+ "id": "sample",+ "meta": "sample",+ "headline": "sample"+ }+ self.assertFalse(correlator.check_rule_validity(rule))++ rule = {+ "id": "sample",+ "meta": "sample",+ "collections": []+ }+ self.assertFalse(correlator.check_rule_validity(rule))