SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name: sfp_searchcode
# Purpose: Search searchcode for code repositories mentioning the target domain.
#
# Author: <[email protected]>
#
# Created: 2021-07-06
# Copyright: (c) bcoles 2021
# Licence: MIT
# -------------------------------------------------------------------------------
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
class sfp_searchcode(SpiderFootPlugin):
meta = {
'name': "searchcode",
'summary': "Search searchcode for code repositories mentioning the target domain.",
'flags': [],
'useCases': ["Investigate", "Footprint", "Passive"],
'categories': ["Search Engines"],
'dataSource': {
'model': "FREE_NOAUTH_UNLIMITED",
'references': [
"https://searchcode.com/api/",
],
'website': "https://searchcode.com/",
'logo': "https://searchcode.com/static/small_logo.png",
'description': "Simple, comprehensive code search."
}
}
opts = {
'max_pages': 10,
'dns_resolve': True,
}
optdescs = {
'max_pages': "Maximum number of pages of results to fetch.",
'dns_resolve': "DNS resolve each identified domain.",
}
results = None
errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
def watchedEvents(self):
return [
'DOMAIN_NAME'
]
def producedEvents(self):
return [
'EMAILADDR',
'EMAILADDR_GENERIC',
'LINKED_URL_INTERNAL',
'PUBLIC_CODE_REPO',
'RAW_RIR_DATA',
]
def query(self, qry, page=1, per_page=100):
params = urllib.parse.urlencode({
'q': qry,
'p': page,
'per_page': per_page
})
res = self.sf.fetchUrl(
f"https://searchcode.com/api/codesearch_I/?{params}",
useragent=self.opts['_useragent'],
timeout=self.opts['_fetchtimeout']
)
time.sleep(2)
if res['content'] is None:
return None
if res['code'] == "429":
self.error("You are being rate-limited by searchcode.")
self.errorState = True
return None
if res['code'] != '200':
self.error(f"Unexpected reply from searchcode: {res['code']}")
self.errorState = True
return None
try:
return json.loads(res['content'])
except Exception as e:
self.debug(f"Error processing JSON response from searchcode: {e}")
return None
return None
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
if self.errorState:
return
self.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventData in self.results:
self.debug(f"Skipping {eventData}, already checked.")
return
if eventName not in self.watchedEvents():
return
self.results[eventData] = True
max_pages = int(self.opts['max_pages'])
page = 0
while page < max_pages:
if self.checkForStop():
return
data = self.query(eventData, page)
page += 1
if not data:
self.errorState = True
return
results = data.get('results')
if not results:
return
emails = SpiderFootHelpers.extractEmailsFromText(str(results))
for email in emails:
if email in self.results:
continue
mail_domain = email.lower().split('@')[1]
if not self.getTarget().matches(mail_domain):
self.debug(f"Skipped email address: {email}")
continue
self.info(f"Found e-mail address: {email}")
evt_type = "EMAILADDR"
if email.split("@")[0] in self.opts['_genericusers'].split(","):
evt_type = "EMAILADDR_GENERIC"
evt = SpiderFootEvent(evt_type, email, self.__name__, event)
self.notifyListeners(evt)
self.results[email] = True
links = set()
for result in results:
lines = result.get('lines')
if lines:
for line in lines:
links.update(SpiderFootHelpers.extractUrlsFromText(lines[line]))
for link in links:
if link in self.results:
continue
host = self.sf.urlFQDN(link)
if not self.getTarget().matches(host, includeChildren=True, includeParents=True):
self.debug(f"Skipped unrelated URL: {link}")
continue
self.debug(f"Found a URL: {link}")
evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event)
self.notifyListeners(evt)
self.results[link] = True
if host in self.results:
continue
if self.opts['dns_resolve'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):
self.debug(f"Host {host} could not be resolved")
evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event)
self.notifyListeners(evt)
self.results[host] = True
for result in results:
if eventData not in str(result):
continue
repo = result.get('repo')
if not repo:
continue
if repo in self.results:
continue
url = result.get('url')
if not url:
continue
repo_data = f"{repo}\n<SFURL>{url}</SFURL>"
evt = SpiderFootEvent('PUBLIC_CODE_REPO', repo_data, self.__name__, event)
self.notifyListeners(evt)
evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(result), self.__name__, event)
self.notifyListeners(evt)
self.results[repo] = True
if not data.get('nextpage'):
break
# End of sfp_searchcode class