SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_google_tag_manager: Add Google Tag Manager (GTM) module (#1817)
bcolese84db07ffa68
diff --git a/modules/sfp_google_tag_manager.py b/modules/sfp_google_tag_manager.pynew file mode 100644index 00000000..55ccbebf--- /dev/null+++ b/modules/sfp_google_tag_manager.py@@ -0,0 +1,184 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_google_tag_manager+# Purpose: Search Google Tag Manager (GTM) for hosts sharing the same GTM code.+#+# Author: <[email protected]>+#+# Created: 2023-10-29+# Copyright: (c) bcoles 2023+# Licence: MIT+# -------------------------------------------------------------------------------++import re+import urllib.error+import urllib.parse+import urllib.request++from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin+++class sfp_google_tag_manager(SpiderFootPlugin):++ meta = {+ 'name': "Google Tag Manager",+ 'summary': "Search Google Tag Manager (GTM) for hosts sharing the same GTM code.",+ 'flags': [],+ 'useCases': ["Footprint", "Investigate", "Passive"],+ 'categories': ["Passive DNS"],+ 'dataSource': {+ 'website': "https://tagmanager.google.com",+ 'model': "FREE_NOAUTH_UNLIMITED",+ 'references': [+ "https://marketingplatform.google.com/about/tag-manager/",+ "https://developers.google.com/tag-manager/quickstart",+ "https://developers.google.com/tag-manager/devguide"+ ],+ 'favIcon': "https://google.com/favicon.ico",+ 'logo': "https://google.com/favicon.ico",+ 'description': "Manage all your website tags without editing code. Google Tag Manager "+ "delivers simple, reliable, easily integrated tag management solutions for free."+ }+ }++ opts = {+ "verify": True+ }++ optdescs = {+ "verify": "Verify identified hostnames resolve to an IP address."+ }++ results = None+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = self.tempStorage()++ for opt in list(userOpts.keys()):+ self.opts[opt] = userOpts[opt]++ def watchedEvents(self):+ return ['WEB_ANALYTICS_ID']++ def producedEvents(self):+ return [+ 'DOMAIN_NAME',+ 'INTERNET_NAME',+ 'AFFILIATE_DOMAIN_NAME',+ 'AFFILIATE_INTERNET_NAME',+ ]++ # from: https://stackoverflow.com/a/43211062+ def is_valid_hostname(self, hostname: str = None) -> bool:+ if not hostname:+ return False+ if len(hostname) > 255:+ return False++ hostname = hostname.rstrip(".")+ allowed = re.compile("(?!-)[A-Z0-9-_]{1,63}(?<!-)$", re.IGNORECASE)+ return all(allowed.match(x) for x in hostname.split("."))++ def queryGoogleTagId(self, tag_id: str = None) -> set:+ if not tag_id:+ return None++ params = urllib.parse.urlencode({+ 'id': tag_id,+ })++ res = self.sf.fetchUrl(+ f"https://googletagmanager.com/gtm.js?{params}",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent']+ )++ if res['code'] != "200":+ self.debug(f"Invalid GTM tag id: {tag_id}")+ return None++ data = res['content']++ if not data:+ self.debug(f"Invalid GTM tag id: {tag_id}")+ return None++ hosts = list()++ for host in re.findall(r'"map","key","(.+?)"', data):+ if '.' not in host:+ continue+ if self.is_valid_hostname(host):+ hosts.append(host)++ for host in re.findall(r',"arg1":"(.+?)"', data):+ if '.' not in host:+ continue+ if self.is_valid_hostname(host):+ hosts.append(host)++ for url in SpiderFootHelpers.extractUrlsFromText(str(data).replace("\\/", "/")):+ host = self.sf.urlFQDN(url)+ if not host:+ continue+ if '.' not in host:+ continue+ hosts.append(host)++ return set(hosts)++ def handleEvent(self, event):+ self.debug(f"Received event, {event.eventType}, from {event.module}")++ if self.errorState:+ return++ if event.data in self.results:+ return++ self.results[event.data] = True++ try:+ network = event.data.split(": ")[0]+ tag_id = event.data.split(": ")[1]+ except Exception as e:+ self.error(f"Unable to parse WEB_ANALYTICS_ID: {event.data} ({e})")+ return++ if network != 'Google Tag Manager':+ return++ hosts = self.queryGoogleTagId(tag_id)++ if not hosts:+ self.info(f"No hosts found for {tag_id}")+ return++ self.info(f"Retrieved {len(hosts)} results")++ for host in hosts:+ # we ignore unresolved hosts due to large number of false positives+ if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):+ self.debug(f"Potential host name '{host}' could not be resolved")+ continue++ if self.getTarget().matches(host, includeChildren=True, includeParents=True):+ evt_type = 'INTERNET_NAME'+ else:+ evt_type = 'AFFILIATE_INTERNET_NAME'++ evt = SpiderFootEvent(evt_type, host, self.__name__, event)+ self.notifyListeners(evt)++ if self.sf.isDomain(host, self.opts['_internettlds']):+ if evt_type.startswith('AFFILIATE'):+ evt_type = 'AFFILIATE_DOMAIN_NAME'+ else:+ evt_type = 'DOMAIN_NAME'++ evt = SpiderFootEvent(evt_type, host, self.__name__, event)+ self.notifyListeners(evt)++# End of sfp_google_tag_manager classdiff --git a/test/integration/modules/test_sfp_google_tag_manager.py b/test/integration/modules/test_sfp_google_tag_manager.pynew file mode 100644index 00000000..2660b938--- /dev/null+++ b/test/integration/modules/test_sfp_google_tag_manager.py@@ -0,0 +1,32 @@+import pytest+import unittest++from modules.sfp_google_tag_manager import sfp_google_tag_manager+from sflib import SpiderFoot+from spiderfoot import SpiderFootEvent, SpiderFootTarget+++class TestModuleIntegrationGoogleTagManager(unittest.TestCase):++ @unittest.skip("todo")+ def test_handleEvent(self):+ sf = SpiderFoot(self.default_options)++ module = sfp_google_tag_manager()+ module.setup(sf, dict())++ target_value = 'example target value'+ target_type = 'IP_ADDRESS'+ target = SpiderFootTarget(target_value, target_type)+ module.setTarget(target)++ event_type = 'ROOT'+ event_data = 'example data'+ event_module = ''+ source_event = ''+ evt = SpiderFootEvent(event_type, event_data, event_module, source_event)++ result = module.handleEvent(evt)++ self.assertIsNone(result)diff --git a/test/unit/modules/test_sfp_google_tag_manager.py b/test/unit/modules/test_sfp_google_tag_manager.pynew file mode 100644index 00000000..3ed3f531--- /dev/null+++ b/test/unit/modules/test_sfp_google_tag_manager.py@@ -0,0 +1,26 @@+import pytest+import unittest++from modules.sfp_google_tag_manager import sfp_google_tag_manager+from sflib import SpiderFoot+++class TestModulesGoogleTagManager(unittest.TestCase):++ def test_opts(self):+ module = sfp_google_tag_manager()+ self.assertEqual(len(module.opts), len(module.optdescs))++ def test_setup(self):+ sf = SpiderFoot(self.default_options)+ module = sfp_google_tag_manager()+ module.setup(sf, dict())++ def test_watchedEvents_should_return_list(self):+ module = sfp_google_tag_manager()+ self.assertIsInstance(module.watchedEvents(), list)++ def test_producedEvents_should_return_list(self):+ module = sfp_google_tag_manager()+ self.assertIsInstance(module.producedEvents(), list)