SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_koodous: Add modules/sfp_koodous.py
Brendan Colesbae7d8e50739
diff --git a/modules/sfp_koodous.py b/modules/sfp_koodous.pynew file mode 100644index 00000000..b4f25c9a--- /dev/null+++ b/modules/sfp_koodous.py@@ -0,0 +1,188 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_koodous+# Purpose: Query Koodous for mobile apps.+#+# Author: <[email protected]>+#+# Created: 2020-09-19+# Copyright: (c) bcoles 2019+# Licence: GPL+# -------------------------------------------------------------------------------++import json+import re+import time+import urllib.error+import urllib.parse+import urllib.request++from spiderfoot import SpiderFootEvent, SpiderFootPlugin+++class sfp_koodous(SpiderFootPlugin):++ meta = {+ 'name': "Koodous",+ 'summary': "Search Koodous for mobile apps.",+ 'flags': [""],+ 'useCases': ["Investigate", "Footprint", "Passive"],+ 'categories': ["Search Engines"],+ 'dataSource': {+ 'model': "FREE_NOAUTH_UNLIMITED",+ 'references': [+ "https://docs.koodous.com/rest-api/apks/",+ ],+ 'website': "https://koodous.com/apks/",+ 'logo': "https://koodous.com/assets/img/koodous-logo.png",+ }+ }++ opts = {+ 'max_pages': 10,+ }++ optdescs = {+ 'max_pages': "Maximum number of pages of results to fetch.",+ }++ results = None+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = self.tempStorage()++ for opt in list(userOpts.keys()):+ self.opts[opt] = userOpts[opt]++ def watchedEvents(self):+ return [+ 'DOMAIN_NAME'+ ]++ def producedEvents(self):+ return [+ 'APPSTORE_ENTRY',+ ]++ def queryPackageName(self, qry, cursor=''):+ package_name = qry.encode('raw_unicode_escape').decode("ascii", errors='replace')++ params = urllib.parse.urlencode({+ 'cursor': cursor,+ 'search': f"package_name:*{package_name}.*"+ })++ res = self.sf.fetchUrl(+ f"https://api.koodous.com/apks?{params}",+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']+ )++ time.sleep(1)++ if res['content'] is None:+ return None++ if res['code'] != '200':+ self.sf.error(f"Unexpected reply from Koodous: {res['code']}")+ self.errorState = True+ return None++ try:+ return json.loads(res['content'])+ except Exception as e:+ self.sf.debug(f"Error processing JSON response from Koodous: {e}")+ return None++ return None++ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return++ self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")++ if eventData in self.results:+ self.sf.debug(f"Skipping {eventData}, already checked.")+ return++ if eventName not in self.watchedEvents():+ return++ self.results[eventData] = True++ domain_reversed = '.'.join(list(reversed(eventData.lower().split('.'))))++ max_pages = int(self.opts['max_pages'])+ page = 1+ cursor = ''+ while page <= max_pages:+ if self.checkForStop():+ return++ data = self.queryPackageName(domain_reversed, cursor)++ page += 1++ if not data:+ self.errorState = True+ return++ evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(data), self.__name__, event)+ self.notifyListeners(evt)++ results = data.get('results')++ for result in results:+ package_name = result.get('package_name')++ if not package_name:+ continue++ app = result.get('app')++ if not app:+ continue++ displayed_version = result.get('displayed_version')++ if not displayed_version:+ continue++ app_full_name = f"{app} {displayed_version} ({package_name})"++ if (+ domain_reversed != package_name.lower()+ and not package_name.lower().startswith(f"{domain_reversed}.")+ and not package_name.lower().endswith(f".{domain_reversed}")+ and f".{domain_reversed}." not in package_name.lower()+ ):+ self.sf.debug(f"App {app_full_name} does not match {domain_reversed}, skipping")+ continue++ sha256 = result.get('sha256')++ if not sha256:+ continue++ app_data = f"{app_full_name}\n<SFURL>https://koodous.com/apks/{sha256}</SFURL>"++ evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event)+ self.notifyListeners(evt)++ if not data.get('next'):+ break++ next_cursor = re.findall('cursor=(.+?)&', data.get('next'))+ if not next_cursor:+ break++ cursor = urllib.parse.unquote(next_cursor[0])++# End of sfp_koodous classdiff --git a/test/unit/modules/test_sfp_koodous.py b/test/unit/modules/test_sfp_koodous.pynew file mode 100644index 00000000..ace67001--- /dev/null+++ b/test/unit/modules/test_sfp_koodous.py@@ -0,0 +1,80 @@+# test_sfp_koodous.py+import unittest++from modules.sfp_koodous import sfp_koodous+from sflib import SpiderFoot+from spiderfoot import SpiderFootEvent, SpiderFootTarget+++class TestModuleKoodous(unittest.TestCase):+ """+ Test modules.sfp_koodous+ """++ default_options = {+ '_debug': False, # Debug+ '__logging': True, # Logging in general+ '__outputfilter': None, # Event types to filter from modules' output+ '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests+ '_dnsserver': '', # Override the default resolver+ '_fetchtimeout': 5, # number of seconds before giving up on a fetch+ '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',+ '_internettlds_cache': 72,+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",+ '__version__': '3.3-DEV',+ '__database': 'spiderfoot.test.db', # note: test database file+ '__modules__': None, # List of modules. Will be set after start-up.+ '_socks1type': '',+ '_socks2addr': '',+ '_socks3port': '',+ '_socks4user': '',+ '_socks5pwd': '',+ '_torctlport': 9051,+ '__logstdout': False+ }++ def test_opts(self):+ module = sfp_koodous()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ """+ Test setup(self, sfc, userOpts=dict())+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_koodous()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_koodous()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_koodous()+ self.assertIsInstance(module.producedEvents(), list)++ @unittest.skip("todo")+ def test_handleEvent(self):+ """+ Test handleEvent(self, event)+ """+ sf = SpiderFoot(self.default_options)++ module = sfp_koodous()+ module.setup(sf, dict())++ target_value = 'example target value'+ target_type = 'IP_ADDRESS'+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = 'ROOT'+ event_data = 'example data'+ event_module = ''+ source_event = ''+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)