SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_malwaredomainlist: Remove sfp_malwaredomainlist module (#1450)
bcoles282dd427e7a7
diff --git a/modules/sfp_malwaredomainlist.py b/modules/sfp_malwaredomainlist.pydeleted file mode 100644index c8352e53..00000000--- a/modules/sfp_malwaredomainlist.py+++ /dev/null@@ -1,263 +0,0 @@-# -*- coding: utf-8 -*--# --------------------------------------------------------------------------------# Name: sfp_malwaredomainlist-# Purpose: Checks if an ASN, IP or domain is malicious.-#-# Author: [email protected]-#-# Created: 14/12/2013-# Copyright: (c) Steve Micallef, 2013-# Licence: GPL-# ---------------------------------------------------------------------------------import re--from netaddr import IPAddress, IPNetwork--from spiderfoot import SpiderFootEvent, SpiderFootPlugin--malchecks = {- 'malwaredomainlist.com IP List': {- 'id': 'malwaredomainlistip',- 'checks': ['ip', 'netblock'],- 'url': 'http://www.malwaredomainlist.com/hostslist/ip.txt'- },- 'malwaredomainlist.com Domain List': {- 'id': 'malwaredomainlistdomain',- 'checks': ['domain'],- 'url': 'http://www.malwaredomainlist.com/hostslist/hosts.txt',- 'regex': r'.*\s+{0}[\s$]'- }-}---class sfp_malwaredomainlist(SpiderFootPlugin):-- meta = {- 'name': "malwaredomainlist.com",- 'summary': "Check if a host/domain, IP or netblock is malicious according to malwaredomainlist.com.",- 'flags': [],- 'useCases': ["Investigate", "Passive"],- 'categories': ["Reputation Systems"],- 'dataSource': {- 'website': "http://www.malwaredomainlist.com/",- 'model': "FREE_NOAUTH_UNLIMITED",- 'references': [- "http://www.malwaredomainlist.com/mdl.php",- "http://www.malwaredomainlist.com/forums/index.php?topic=3270.0"- ],- 'favIcon': "http://www.malwaredomainlist.com/favicon.ico",- 'logo': "http://www.malwaredomainlist.com/favicon.ico",- 'description': "Analyzing malicious websites? Malware Domain List is a non-commercial community project. "- "Our list can be used for free by anyone. Feel free to use it. "- "You can find an overview of downloadable lists here.",- }- }-- # Default options- opts = {- 'malwaredomainlistdomain': True,- 'malwaredomainlistip': True,- 'checkaffiliates': True,- 'checkcohosts': True,- 'cacheperiod': 18,- 'checknetblocks': True,- 'checksubnets': True- }-- # Option descriptions- optdescs = {- 'malwaredomainlistdomain': "Enable malwaredomainlist.com domain check?",- 'malwaredomainlistip': "Enable malwaredomainlist.com IP check?",- 'checkaffiliates': "Apply checks to affiliates?",- 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?",- 'cacheperiod': "Hours to cache list data before re-fetching.",- 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?",- 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?"- }-- # Be sure to completely clear any class variables in setup()- # or you run the risk of data persisting between scan runs.-- results = None-- def setup(self, sfc, userOpts=dict()):- self.sf = sfc- self.results = self.tempStorage()-- # Clear / reset any other class member variables here- # or you risk them persisting between threads.-- for opt in list(userOpts.keys()):- self.opts[opt] = userOpts[opt]-- # What events is this module interested in for input- # * = be notified about all events.- def watchedEvents(self):- return ["INTERNET_NAME", "IP_ADDRESS", "BGP_AS_OWNER", "BGP_AS_MEMBER",- "NETBLOCK_MEMBER", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR",- "CO_HOSTED_SITE", "NETBLOCK_OWNER"]-- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.- def producedEvents(self):- return ["MALICIOUS_ASN", "MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",- "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME",- "MALICIOUS_SUBNET", "MALICIOUS_COHOST", "MALICIOUS_NETBLOCK"]-- # Look up 'list' type resources- def resourceList(self, replaceme_id, target, targetType):- targetDom = ''- # Get the base domain if we're supplied a domain- if targetType == "domain":- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])- if not targetDom:- return None-- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if replaceme_id == cid:- data = dict()- url = malchecks[check]['url']- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))- if data['content'] is None:- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if data['content'] is None:- self.sf.error("Unable to fetch " + url)- return None- else:- self.sf.cachePut("sfmal_" + cid, data['content'])-- # If we're looking at netblocks- if targetType == "netblock":- iplist = list()- # Get the regex, replace {0} with an IP address matcher to- # build a list of IP.- # Cycle through each IP and check if it's in the netblock.- if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")- pat = re.compile(rx, re.IGNORECASE)- self.sf.debug("New regex for " + check + ": " + rx)- for line in data['content'].split('\n'):- grp = re.findall(pat, line)- if len(grp) > 0:- # self.sf.debug("Adding " + grp[0] + " to list.")- iplist.append(grp[0])- else:- iplist = data['content'].split('\n')-- for ip in iplist:- if len(ip) < 8 or ip.startswith("#"):- continue- ip = ip.strip()-- try:- if IPAddress(ip) in IPNetwork(target):- self.sf.debug(ip + " found within netblock/subnet "- + target + " in " + check)- return url- except Exception as e:- self.sf.debug(f"Error encountered parsing: {e}")- continue-- return None-- # If we're looking at hostnames/domains/IPs- if 'regex' not in malchecks[check]:- for line in data['content'].split('\n'):- if line == target or (targetType == "domain" and line == targetDom):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- else:- # Check for the domain and the hostname- try:- rxDom = str(malchecks[check]['regex']).format(targetDom)- rxTgt = str(malchecks[check]['regex']).format(target)- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- except Exception as e:- self.sf.debug("Error encountered parsing 2: " + str(e))- continue-- return None-- def lookupItem(self, resourceId, itemType, target):- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if cid == resourceId and itemType in malchecks[check]['checks']:- self.sf.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid)- return self.resourceList(cid, target, itemType)-- return None-- # Handle events sent to this module- def handleEvent(self, event):- eventName = event.eventType- srcModuleName = event.module- eventData = event.data-- self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")-- if eventData in self.results:- self.sf.debug(f"Skipping {eventData}, already checked.")- return-- self.results[eventData] = True-- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):- return- if eventName == 'AFFILIATE_IPADDR' \- and not self.opts.get('checkaffiliates', False):- return- if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False):- return- if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False):- return-- for check in list(malchecks.keys()):- cid = malchecks[check]['id']-- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:- typeId = 'ip'- if eventName == 'IP_ADDRESS':- evtType = 'MALICIOUS_IPADDR'- else:- evtType = 'MALICIOUS_AFFILIATE_IPADDR'-- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:- typeId = 'asn'- evtType = 'MALICIOUS_ASN'-- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',- 'AFFILIATE_INTERNET_NAME']:- typeId = 'domain'- if eventName == "INTERNET_NAME":- evtType = "MALICIOUS_INTERNET_NAME"- if eventName == 'AFFILIATE_INTERNET_NAME':- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'- if eventName == 'CO_HOSTED_SITE':- evtType = 'MALICIOUS_COHOST'-- if eventName == 'NETBLOCK_OWNER':- typeId = 'netblock'- evtType = 'MALICIOUS_NETBLOCK'- if eventName == 'NETBLOCK_MEMBER':- typeId = 'netblock'- evtType = 'MALICIOUS_SUBNET'-- url = self.lookupItem(cid, typeId, eventData)-- if self.checkForStop():- return-- # Notify other modules of what you've found- if url is not None:- text = f"{check} [{eventData}]\n<SFURL>{url}</SFURL>"- evt = SpiderFootEvent(evtType, text, self.__name__, event)- self.notifyListeners(evt)--# End of sfp_malwaredomainlist classdiff --git a/test/unit/modules/test_sfp_malwaredomainlist.py b/test/unit/modules/test_sfp_malwaredomainlist.pydeleted file mode 100644index b3bd2ada..00000000--- a/test/unit/modules/test_sfp_malwaredomainlist.py+++ /dev/null@@ -1,60 +0,0 @@-# test_sfp_malwaredomainlist.py-import pytest-import unittest--from modules.sfp_malwaredomainlist import sfp_malwaredomainlist-from sflib import SpiderFoot-from spiderfoot import SpiderFootEvent, SpiderFootTarget---class TestModulemalwaredomainlist(unittest.TestCase):- """- Test modules.sfp_malwaredomainlist- """-- def test_opts(self):- module = sfp_malwaredomainlist()- self.assertEqual(len(module.opts), len(module.optdescs))-- def test_setup(self):- """- Test setup(self, sfc, userOpts=dict())- """- sf = SpiderFoot(self.default_options)-- module = sfp_malwaredomainlist()- module.setup(sf, dict())-- def test_watchedEvents_should_return_list(self):- module = sfp_malwaredomainlist()- self.assertIsInstance(module.watchedEvents(), list)-- def test_producedEvents_should_return_list(self):- module = sfp_malwaredomainlist()- self.assertIsInstance(module.producedEvents(), list)-- @unittest.skip("todo")- def test_handleEvent(self):- """- Test handleEvent(self, event)- """- sf = SpiderFoot(self.default_options)-- module = sfp_malwaredomainlist()- module.setup(sf, dict())-- target_value = 'example target value'- target_type = 'IP_ADDRESS'- target = SpiderFootTarget(target_value, target_type)- module.setTarget(target)-- event_type = 'ROOT'- event_data = 'example data'- event_module = ''- source_event = ''- evt = SpiderFootEvent(event_type, event_data, event_module, source_event)-- result = module.handleEvent(evt)-- self.assertIsNone(result)