SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Massive reduction in false positives for junk files through better validation.
smicallef474da5e8e7cc
diff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.pyindex 95988e22..1308b928 100644--- a/modules/sfp_junkfiles.py+++ b/modules/sfp_junkfiles.py@@ -36,17 +36,17 @@ class sfp_junkfiles(SpiderFootPlugin):'skipfake': "Try to fetch an obviously fake page and if no 404 is returned, stop trying that particular host for junk files. Good for avoiding false positives in cases where servers return content for pages that don't exist."}- results = list()- hosts = list()- skiphosts = list()- bases = list()+ results = dict()+ hosts = dict()+ skiphosts = dict()+ bases = dict()def setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()- self.hosts = list()- self.skiphosts = list()- self.bases = list()+ self.results = dict()+ self.hosts = dict()+ self.skiphosts = dict()+ self.bases = dict()self.__dataSource__ = "Target Website"for opt in userOpts.keys():@@ -62,6 +62,19 @@ class sfp_junkfiles(SpiderFootPlugin):def producedEvents(self):return ["JUNK_FILE"]+ # Test how trustworthy a result is+ def checkValidity(self, junkUrl):+ # Try and fetch an obviously missing version of the junk file+ fetch = junkUrl + str(random.randint(0, 99999999))+ res = self.sf.fetchUrl(fetch, headOnly=True,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])+ if res['code'] != "404":+ host = self.sf.urlBaseUrl(junkUrl)+ self.skiphosts[host] = True+ return False+ return True+# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventType@@ -74,27 +87,18 @@ class sfp_junkfiles(SpiderFootPlugin):if eventData in self.results:return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Trueif self.opts['skipfake'] and host in self.skiphosts:self.sf.debug("Skipping " + host + " because it doesn't return 404s.")return None- # Try and fetch an obviously missing page- if host not in self.hosts and self.opts['skipfake']:- fetch = host + "/" + str(random.randint(0, 99999999)) + ".html"- res = self.sf.fetchUrl(fetch, headOnly=True,- timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])- if res['code'] != "404":- self.skiphosts.append(host)- self.hosts.append(host)- return None-- self.hosts.append(host)-# http://www/blah/abc.php -> try http://www/blah/abc.php.[fileexts]for ext in self.opts['urlextstry']:+ if self.opts['skipfake'] and host in self.skiphosts:+ self.sf.debug("Skipping " + host + " because it doesn't return 404s.")+ return None+if "." + ext + "?" in eventData or "." + ext + "#" in eventData or \eventData.endswith("." + ext):bits = eventData.split("?")@@ -105,7 +109,7 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Trying " + x + " against " + eventData)fetch = bits[0] + "." + xif fetch not in self.results:- self.results.append(fetch)+ self.results[fetch] = Trueelse:self.sf.debug("Skipping, already fetched.")continue@@ -117,15 +121,17 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch)continueif res['code'] == "200":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)+ if not self.checkValidity(fetch):+ continue++ evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)self.notifyListeners(evt)base = self.sf.urlBaseDir(eventData)if base in self.bases:return Noneelse:- self.bases.append(base)+ self.bases[base] = True# don't do anything with the root directory of a siteself.sf.debug("Base: " + base + ", event: " + eventData)@@ -137,6 +143,10 @@ class sfp_junkfiles(SpiderFootPlugin):if self.checkForStop():return None+ if self.opts['skipfake'] and host in self.skiphosts:+ self.sf.debug("Skipping " + host + " because it doesn't return 404s.")+ return None+if base.count('/') == 3:self.sf.debug("Skipping base url.")continue@@ -144,7 +154,7 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Trying " + dirfile + " against " + eventData)fetch = base[0:len(base) - 1] + "." + dirfileif fetch not in self.results:- self.results.append(fetch)+ self.results[fetch] = Trueelse:self.sf.debug("Skipping, already fetched.")continue@@ -155,8 +165,10 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch)continueif res['code'] == "200":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)+ if not self.checkValidity(fetch):+ continue++ evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)self.notifyListeners(evt)# http://www/blah/abc.html -> try http://www/blah/[files]@@ -164,10 +176,14 @@ class sfp_junkfiles(SpiderFootPlugin):if self.checkForStop():return None+ if self.opts['skipfake'] and host in self.skiphosts:+ self.sf.debug("Skipping " + host + " because it doesn't return 404s.")+ return None+self.sf.debug("Trying " + f + " against " + eventData)fetch = base + fif fetch not in self.results:- self.results.append(fetch)+ self.results[fetch] = Trueelse:self.sf.debug("Skipping, already fetched.")continue@@ -178,8 +194,10 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch)continueif res['code'] == "200":- evt = SpiderFootEvent("JUNK_FILE", fetch,- self.__name__, event)+ if not self.checkValidity(fetch):+ continue++ evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)self.notifyListeners(evt)return None