SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Junk file checker module now much smarter, to avoid false positives.
Steve0cfd53cea5a6
diff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.pyindex f99e1dc1..e2fbd70d 100644--- a/modules/sfp_junkfiles.py+++ b/modules/sfp_junkfiles.py@@ -11,7 +11,8 @@# -------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-+import hashlib+import randomclass sfp_junkfiles(SpiderFootPlugin):"""Junk Files:Footprint:Looks for old/temporary and other similar files."""@@ -23,7 +24,8 @@ class sfp_junkfiles(SpiderFootPlugin):'files': ['x', 'xxx', 'crap', 'old', 'a', 'aaa', 'z', 'zzz','out', 'sql', "passwd", ".htaccess", ".htpasswd","Thumbs.db", "asd", "asdf"],- 'dirs': ['zip', 'tar.gz', 'tgz', 'tar']+ 'dirs': ['zip', 'tar.gz', 'tgz', 'tar'],+ 'skipfake': True}# Option descriptions@@ -31,14 +33,21 @@ class sfp_junkfiles(SpiderFootPlugin):'fileexts': "File extensions to try.",'urlextstry': "Try those extensions against URLs with these extensions.",'files': "Try to fetch each of these files from the directory of the URL.",- 'dirs': "Try to fetch the containing folder with these extensions."+ 'dirs': "Try to fetch the containing folder with these extensions.",+ 'skipfake': "Try to fetch an obviously fake page and if no 404 is returned, stop trying that particular host for junk files. Good for avoiding false positives in cases where servers return content for pages that don't exist."}results = list()+ contentHashes = list()+ hosts = list()+ skiphosts = list()def setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = list()+ self.contentHashes = list()+ self.hosts = list()+ self.skiphosts = list()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -53,11 +62,22 @@ class sfp_junkfiles(SpiderFootPlugin):def producedEvents(self):return ["JUNK_FILE"]+ # Check if contents are standard contents+ def isGeneric(self, content):+ hashStr = hashlib.sha1(content.encode('raw_unicode_escape')).hexdigest()+ if hashStr in self.contentHashes:+ self.sf.debug("Skipping due to looking like a generic page.")+ return True+ else:+ self.contentHashes.append(hashStr)+ return False+# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data+ host = self.sf.urlBaseUrl(eventData)self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)@@ -66,6 +86,23 @@ class sfp_junkfiles(SpiderFootPlugin):else:self.results.append(eventData)+ if host in self.skiphosts:+ return None++ # Try and fetch an obviously missing page, store the hash+ if host not in self.hosts:+ res = self.sf.fetchUrl(host + "/" + str(random.randint(0, 99999999)) + ".html",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])+ if res['content'] != None:+ hashStr = hashlib.sha1(res['content'].encode('raw_unicode_escape')).hexdigest()+ self.contentHashes.append(hashStr)+ if self.opts['skipfake']:+ skiphosts.append(host)+ return None++ self.hosts.append(host)+# http://www/blah/abc.html -> try http://www/blah/abc.html.[fileexts]for ext in self.opts['urlextstry']:if "." + ext + "?" in eventData or "." + ext + "#" in eventData or \@@ -80,14 +117,16 @@ class sfp_junkfiles(SpiderFootPlugin):if fetch not in self.results:self.results.append(fetch)else:+ self.sf.debug("Skipping, already fetched.")continueres = self.sf.fetchUrl(fetch,timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])- if res['content'] is not None and res['code'] != "404":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)- self.notifyListeners(evt)+ if res['content'] is not None and res['code'] in ["200", 200]:+ if not self.isGeneric(res['content']):+ evt = SpiderFootEvent("JUNK_FILE", fetch,+ self.__name__, event)+ self.notifyListeners(evt)# http://www or http://www/# -> try index pages@@ -102,17 +141,20 @@ class sfp_junkfiles(SpiderFootPlugin):if fetch not in self.results:self.results.append(fetch)else:+ self.sf.debug("Skipping, already fetched.")continueres = self.sf.fetchUrl(fetch,timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])- if res['content'] is not None and res['code'] != "404":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)- self.notifyListeners(evt)+ if res['content'] is not None and res['code'] in [ "200", 200]:+ if not self.isGeneric(res['content']):+ evt = SpiderFootEvent("JUNK_FILE", fetch,+ self.__name__, event)+ self.notifyListeners(evt)base = self.sf.urlBaseDir(eventData)# don't do anything with the root directory of a site+ self.sf.debug("Base: " + base + ", event: " + eventData)if base == eventData + "/" or base == eventData:return None@@ -121,19 +163,25 @@ class sfp_junkfiles(SpiderFootPlugin):if self.checkForStop():return None+ if base.count('/') < 4:+ self.sf.debug("Skipping base url.")+ continue+self.sf.debug("Trying " + dirfile + " against " + eventData)fetch = base[0:len(base) - 1] + "." + dirfileif fetch not in self.results:self.results.append(fetch)else:+ self.sf.debug("Skipping, already fetched.")continueres = self.sf.fetchUrl(fetch,timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])- if res['content'] is not None and res['code'] != "404":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)- self.notifyListeners(evt)+ if res['content'] is not None and res['code'] in ["200", 200]:+ if not self.isGeneric(res['content']):+ evt = SpiderFootEvent("JUNK_FILE", fetch,+ self.__name__, event)+ self.notifyListeners(evt)# http://www/blah/abc.html -> try http://www/blah/[files]for f in self.opts['files']:@@ -145,14 +193,16 @@ class sfp_junkfiles(SpiderFootPlugin):if fetch not in self.results:self.results.append(fetch)else:+ self.sf.debug("Skipping, already fetched.")continueres = self.sf.fetchUrl(fetch,timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])- if res['content'] is not None and res['code'] != "404":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)- self.notifyListeners(evt)+ if res['content'] is not None and res['code'] in ["200", 200]:+ if not self.isGeneric(res['content']):+ evt = SpiderFootEvent("JUNK_FILE", fetch,+ self.__name__, event)+ self.notifyListeners(evt)return None