Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

10.7 KB
Raw Blame
# -------------------------------------------------------------------------------
# Name:         sfp_builtwith
# Purpose:      Query builtwith.com using their API.
#
# Author:      Steve Micallef <[email protected]>
#
# Created:     10/08/2017
# Copyright:   (c) Steve Micallef
# Licence:     MIT
# -------------------------------------------------------------------------------

import json
import time

from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin


class sfp_builtwith(SpiderFootPlugin):

    meta = {
        'name': "BuiltWith",
        'summary': "Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.",
        'flags': ["apikey"],
        'useCases': ["Footprint", "Investigate", "Passive"],
        'categories': ["Search Engines"],
        'dataSource': {
            'website': "https://builtwith.com/",
            'model': "FREE_AUTH_LIMITED",
            'references': [
                "https://api.builtwith.com/",
                "https://kb.builtwith.com/",
                "https://builtwith.com/screencast",
                "https://builtwith.com/faq"
            ],
            'apiKeyInstructions': [
                "Visit https://api.builtwith.com/free-api",
                "Register a free account",
                "Navigate to https://api.builtwith.com/free-api",
                "The API key is listed under 'Your API Key'"
            ],
            'favIcon': "https://d28rh9vvmrd65v.cloudfront.net/favicon.ico",
            'logo': "https://d28rh9vvmrd65v.cloudfront.net/favicon.ico",
            'description': "Build lists of websites from our database of 38,701+ web technologies and over a quarter of a billion websites showing "
            "which sites use shopping carts, analytics, hosting and many more. "
            "Filter by location, traffic, vertical and more.\n"
            "Know your prospects platform before you talk to them. "
            "Improve your conversions with validated market adoption.\n"
            "Get advanced technology market share information and country based analytics for all web technologies.",
        }
    }

    # Default options
    opts = {
        "api_key": "",
        "maxage": 30
    }

    # Option descriptions
    optdescs = {
        "api_key": "Builtwith.com Domain API key.",
        "maxage": "The maximum age of the data returned, in days, in order to be considered valid."
    }

    # Be sure to completely clear any class variables in setup()
    # or you run the risk of data persisting between scan runs.

    results = None
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = self.tempStorage()
        self.errorState = False

        # Clear / reset any other class member variables here
        # or you risk them persisting between threads.

        for opt in list(userOpts.keys()):
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME"]

    # What events this module produces
    def producedEvents(self):
        return ["INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
                "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME",
                "CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID"]

    def queryRelationships(self, t):
        url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}"

        res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")

        if res['code'] == "404":
            return None

        if not res['content']:
            return None

        try:
            return json.loads(res['content'])['Relationships']
        except Exception as e:
            self.error(f"Error processing JSON response from builtwith.com: {e}")

        return None

    def queryDomainInfo(self, t):
        url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}"

        res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")

        if res['code'] == "404":
            return None

        if not res['content']:
            return None

        try:
            return json.loads(res['content'])['Results'][0]
        except Exception as e:
            self.error(f"Error processing JSON response from builtwith.com: {e}")

        return None

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if self.errorState:
            return

        self.debug(f"Received event, {eventName}, from {srcModuleName}")

        if self.opts['api_key'] == "":
            self.error("You enabled sfp_builtwith but did not set an API key!")
            self.errorState = True
            return

        if eventData in self.results:
            self.debug(f"Skipping {eventData}, already checked.")
            return

        self.results[eventData] = True

        data = self.queryDomainInfo(eventData)
        if data is not None:
            if "Meta" in data:
                if data['Meta'].get("Names", []):
                    for nb in data['Meta']['Names']:
                        e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + nb['Name'],
                                            self.__name__, event)
                        self.notifyListeners(e)
                        if nb.get('Email', None):
                            if SpiderFootHelpers.validEmail(nb['Email']):
                                if nb['Email'].split("@")[0] in self.opts['_genericusers'].split(","):
                                    evttype = "EMAILADDR_GENERIC"
                                else:
                                    evttype = "EMAILADDR"
                                e = SpiderFootEvent(evttype, nb['Email'],
                                                    self.__name__, event)
                                self.notifyListeners(e)

                if data['Meta'].get("Emails", []):
                    for email in data['Meta']['Emails']:
                        if SpiderFootHelpers.validEmail(email):
                            if email.split("@")[0] in self.opts['_genericusers'].split(","):
                                evttype = "EMAILADDR_GENERIC"
                            else:
                                evttype = "EMAILADDR"

                            e = SpiderFootEvent(evttype, email,
                                                self.__name__, event)
                            self.notifyListeners(e)

                if data['Meta'].get("Telephones", []):
                    for phone in data['Meta']['Telephones']:
                        phone = phone.replace("-", "").replace("(", "").replace(")", "").replace(" ", "")
                        e = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event)
                        self.notifyListeners(e)

            if "Paths" in data.get("Result", []):
                for p in data["Result"]['Paths']:
                    if p.get("SubDomain", ""):
                        h = p["SubDomain"] + "." + eventData
                        ev = SpiderFootEvent("INTERNET_NAME", h, self.__name__, event)
                        self.notifyListeners(ev)
                        if self.sf.isDomain(h, self.opts['_internettlds']):
                            ev = SpiderFootEvent("DOMAIN_NAME", h, self.__name__, event)
                            self.notifyListeners(ev)
                    else:
                        ev = None

                    # If we have a subdomain, let's get its tech info
                    # and associate it with the subdomain event.
                    for t in p.get("Technologies", []):
                        if ev:
                            src = ev
                        else:
                            src = event
                        agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])
                        if t.get("LastDetected", 0) < agelimit:
                            self.debug("Data found too old, skipping.")
                            continue
                        e = SpiderFootEvent("WEBSERVER_TECHNOLOGY", t["Name"],
                                            self.__name__, src)
                        self.notifyListeners(e)

        data = self.queryRelationships(eventData)
        if data is None:
            return

        agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])

        for r in data:
            if "Domain" not in r or "Identifiers" not in r:
                self.debug("Data returned not in the format requested.")
                continue

            if r['Domain'] != eventData:
                self.debug("Data returned doesn't match data requested, skipping.")
                continue

            for i in r['Identifiers']:
                if "Last" not in i or "Type" not in i or "Value" not in i:
                    self.debug("Data returned not in the format requested.")
                    continue

                if i['Last'] < agelimit:
                    self.debug("Data found too old, skipping.")
                    continue

                evttype = None
                # Related through shared IP
                if i['Type'] == "ip":
                    if self.sf.validIP(i['Value']):
                        val = i['Value']
                        evttype = "IP_ADDRESS"
                    else:
                        val = i['Value'].strip(".")
                        if self.getTarget.matches(val):
                            evttype = "INTERNET_NAME"
                        else:
                            evttype = "CO_HOSTED_SITE"

                    # Create the name/co-host
                    e = SpiderFootEvent(evttype, val, self.__name__, event)
                    self.notifyListeners(e)
                    continue

                # Related through shared analytics ID
                txt = i['Type'] + ": " + str(i['Value'])
                e = SpiderFootEvent("WEB_ANALYTICS_ID", txt, self.__name__, event)
                self.notifyListeners(e)

                if i['Matches']:
                    for m in i['Matches']:
                        if "Domain" not in m:
                            continue
                        evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME", m['Domain'], self.__name__, e)
                        self.notifyListeners(evt)

                        if self.sf.isDomain(m['Domain'], self.opts['_internettlds']):
                            evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", m['Domain'], self.__name__, e)
                            self.notifyListeners(evt)

# End of sfp_builtwith class