Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_malwaredomainlist: Remove sfp_malwaredomainlist module (#1450)

bcoles282dd427e7a7
diff --git a/modules/sfp_malwaredomainlist.py b/modules/sfp_malwaredomainlist.py
deleted file mode 100644
index c8352e53..00000000
--- a/modules/sfp_malwaredomainlist.py
+++ /dev/null
@@ -1,263 +0,0 @@
-# -*- coding: utf-8 -*-
-# -------------------------------------------------------------------------------
-# Name: sfp_malwaredomainlist
-# Purpose: Checks if an ASN, IP or domain is malicious.
-#
-#
-# Created: 14/12/2013
-# Copyright: (c) Steve Micallef, 2013
-# Licence: GPL
-# -------------------------------------------------------------------------------
-
-import re
-
-from netaddr import IPAddress, IPNetwork
-
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin
-
-malchecks = {
- 'malwaredomainlist.com IP List': {
- 'id': 'malwaredomainlistip',
- 'checks': ['ip', 'netblock'],
- 'url': 'http://www.malwaredomainlist.com/hostslist/ip.txt'
- },
- 'malwaredomainlist.com Domain List': {
- 'id': 'malwaredomainlistdomain',
- 'checks': ['domain'],
- 'url': 'http://www.malwaredomainlist.com/hostslist/hosts.txt',
- 'regex': r'.*\s+{0}[\s$]'
- }
-}
-
-
-class sfp_malwaredomainlist(SpiderFootPlugin):
-
- meta = {
- 'name': "malwaredomainlist.com",
- 'summary': "Check if a host/domain, IP or netblock is malicious according to malwaredomainlist.com.",
- 'flags': [],
- 'useCases': ["Investigate", "Passive"],
- 'categories': ["Reputation Systems"],
- 'dataSource': {
- 'website': "http://www.malwaredomainlist.com/",
- 'model': "FREE_NOAUTH_UNLIMITED",
- 'references': [
- "http://www.malwaredomainlist.com/mdl.php",
- "http://www.malwaredomainlist.com/forums/index.php?topic=3270.0"
- ],
- 'favIcon': "http://www.malwaredomainlist.com/favicon.ico",
- 'logo': "http://www.malwaredomainlist.com/favicon.ico",
- 'description': "Analyzing malicious websites? Malware Domain List is a non-commercial community project. "
- "Our list can be used for free by anyone. Feel free to use it. "
- "You can find an overview of downloadable lists here.",
- }
- }
-
- # Default options
- opts = {
- 'malwaredomainlistdomain': True,
- 'malwaredomainlistip': True,
- 'checkaffiliates': True,
- 'checkcohosts': True,
- 'cacheperiod': 18,
- 'checknetblocks': True,
- 'checksubnets': True
- }
-
- # Option descriptions
- optdescs = {
- 'malwaredomainlistdomain': "Enable malwaredomainlist.com domain check?",
- 'malwaredomainlistip': "Enable malwaredomainlist.com IP check?",
- 'checkaffiliates': "Apply checks to affiliates?",
- 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?",
- 'cacheperiod': "Hours to cache list data before re-fetching.",
- 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?",
- 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?"
- }
-
- # Be sure to completely clear any class variables in setup()
- # or you run the risk of data persisting between scan runs.
-
- results = None
-
- def setup(self, sfc, userOpts=dict()):
- self.sf = sfc
- self.results = self.tempStorage()
-
- # Clear / reset any other class member variables here
- # or you risk them persisting between threads.
-
- for opt in list(userOpts.keys()):
- self.opts[opt] = userOpts[opt]
-
- # What events is this module interested in for input
- # * = be notified about all events.
- def watchedEvents(self):
- return ["INTERNET_NAME", "IP_ADDRESS", "BGP_AS_OWNER", "BGP_AS_MEMBER",
- "NETBLOCK_MEMBER", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR",
- "CO_HOSTED_SITE", "NETBLOCK_OWNER"]
-
- # What events this module produces
- # This is to support the end user in selecting modules based on events
- # produced.
- def producedEvents(self):
- return ["MALICIOUS_ASN", "MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",
- "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME",
- "MALICIOUS_SUBNET", "MALICIOUS_COHOST", "MALICIOUS_NETBLOCK"]
-
- # Look up 'list' type resources
- def resourceList(self, replaceme_id, target, targetType):
- targetDom = ''
- # Get the base domain if we're supplied a domain
- if targetType == "domain":
- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])
- if not targetDom:
- return None
-
- for check in list(malchecks.keys()):
- cid = malchecks[check]['id']
- if replaceme_id == cid:
- data = dict()
- url = malchecks[check]['url']
- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))
- if data['content'] is None:
- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
- if data['content'] is None:
- self.sf.error("Unable to fetch " + url)
- return None
- else:
- self.sf.cachePut("sfmal_" + cid, data['content'])
-
- # If we're looking at netblocks
- if targetType == "netblock":
- iplist = list()
- # Get the regex, replace {0} with an IP address matcher to
- # build a list of IP.
- # Cycle through each IP and check if it's in the netblock.
- if 'regex' in malchecks[check]:
- rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")
- pat = re.compile(rx, re.IGNORECASE)
- self.sf.debug("New regex for " + check + ": " + rx)
- for line in data['content'].split('\n'):
- grp = re.findall(pat, line)
- if len(grp) > 0:
- # self.sf.debug("Adding " + grp[0] + " to list.")
- iplist.append(grp[0])
- else:
- iplist = data['content'].split('\n')
-
- for ip in iplist:
- if len(ip) < 8 or ip.startswith("#"):
- continue
- ip = ip.strip()
-
- try:
- if IPAddress(ip) in IPNetwork(target):
- self.sf.debug(ip + " found within netblock/subnet "
- + target + " in " + check)
- return url
- except Exception as e:
- self.sf.debug(f"Error encountered parsing: {e}")
- continue
-
- return None
-
- # If we're looking at hostnames/domains/IPs
- if 'regex' not in malchecks[check]:
- for line in data['content'].split('\n'):
- if line == target or (targetType == "domain" and line == targetDom):
- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
- return url
- else:
- # Check for the domain and the hostname
- try:
- rxDom = str(malchecks[check]['regex']).format(targetDom)
- rxTgt = str(malchecks[check]['regex']).format(target)
- for line in data['content'].split('\n'):
- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
- re.match(rxTgt, line, re.IGNORECASE):
- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
- return url
- except Exception as e:
- self.sf.debug("Error encountered parsing 2: " + str(e))
- continue
-
- return None
-
- def lookupItem(self, resourceId, itemType, target):
- for check in list(malchecks.keys()):
- cid = malchecks[check]['id']
- if cid == resourceId and itemType in malchecks[check]['checks']:
- self.sf.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid)
- return self.resourceList(cid, target, itemType)
-
- return None
-
- # Handle events sent to this module
- def handleEvent(self, event):
- eventName = event.eventType
- srcModuleName = event.module
- eventData = event.data
-
- self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
-
- if eventData in self.results:
- self.sf.debug(f"Skipping {eventData}, already checked.")
- return
-
- self.results[eventData] = True
-
- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):
- return
- if eventName == 'AFFILIATE_IPADDR' \
- and not self.opts.get('checkaffiliates', False):
- return
- if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False):
- return
- if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False):
- return
-
- for check in list(malchecks.keys()):
- cid = malchecks[check]['id']
-
- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:
- typeId = 'ip'
- if eventName == 'IP_ADDRESS':
- evtType = 'MALICIOUS_IPADDR'
- else:
- evtType = 'MALICIOUS_AFFILIATE_IPADDR'
-
- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:
- typeId = 'asn'
- evtType = 'MALICIOUS_ASN'
-
- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',
- 'AFFILIATE_INTERNET_NAME']:
- typeId = 'domain'
- if eventName == "INTERNET_NAME":
- evtType = "MALICIOUS_INTERNET_NAME"
- if eventName == 'AFFILIATE_INTERNET_NAME':
- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'
- if eventName == 'CO_HOSTED_SITE':
- evtType = 'MALICIOUS_COHOST'
-
- if eventName == 'NETBLOCK_OWNER':
- typeId = 'netblock'
- evtType = 'MALICIOUS_NETBLOCK'
- if eventName == 'NETBLOCK_MEMBER':
- typeId = 'netblock'
- evtType = 'MALICIOUS_SUBNET'
-
- url = self.lookupItem(cid, typeId, eventData)
-
- if self.checkForStop():
- return
-
- # Notify other modules of what you've found
- if url is not None:
- text = f"{check} [{eventData}]\n<SFURL>{url}</SFURL>"
- evt = SpiderFootEvent(evtType, text, self.__name__, event)
- self.notifyListeners(evt)
-
-# End of sfp_malwaredomainlist class
diff --git a/test/unit/modules/test_sfp_malwaredomainlist.py b/test/unit/modules/test_sfp_malwaredomainlist.py
deleted file mode 100644
index b3bd2ada..00000000
--- a/test/unit/modules/test_sfp_malwaredomainlist.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# test_sfp_malwaredomainlist.py
-import pytest
-import unittest
-
-from modules.sfp_malwaredomainlist import sfp_malwaredomainlist
-from sflib import SpiderFoot
-from spiderfoot import SpiderFootEvent, SpiderFootTarget
-
-
-class TestModulemalwaredomainlist(unittest.TestCase):
- """
- Test modules.sfp_malwaredomainlist
- """
-
- def test_opts(self):
- module = sfp_malwaredomainlist()
- self.assertEqual(len(module.opts), len(module.optdescs))
-
- def test_setup(self):
- """
- Test setup(self, sfc, userOpts=dict())
- """
- sf = SpiderFoot(self.default_options)
-
- module = sfp_malwaredomainlist()
- module.setup(sf, dict())
-
- def test_watchedEvents_should_return_list(self):
- module = sfp_malwaredomainlist()
- self.assertIsInstance(module.watchedEvents(), list)
-
- def test_producedEvents_should_return_list(self):
- module = sfp_malwaredomainlist()
- self.assertIsInstance(module.producedEvents(), list)
-
- @unittest.skip("todo")
- def test_handleEvent(self):
- """
- Test handleEvent(self, event)
- """
- sf = SpiderFoot(self.default_options)
-
- module = sfp_malwaredomainlist()
- module.setup(sf, dict())
-
- target_value = 'example target value'
- target_type = 'IP_ADDRESS'
- target = SpiderFootTarget(target_value, target_type)
- module.setTarget(target)
-
- event_type = 'ROOT'
- event_data = 'example data'
- event_module = ''
- source_event = ''
- evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
-
- result = module.handleEvent(evt)
-
- self.assertIsNone(result)