Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Massive reduction in false positives for junk files through better validation.

smicallef474da5e8e7cc
diff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.py
index 95988e22..1308b928 100644
--- a/modules/sfp_junkfiles.py
+++ b/modules/sfp_junkfiles.py
@@ -36,17 +36,17 @@ class sfp_junkfiles(SpiderFootPlugin):
'skipfake': "Try to fetch an obviously fake page and if no 404 is returned, stop trying that particular host for junk files. Good for avoiding false positives in cases where servers return content for pages that don't exist."
}
- results = list()
- hosts = list()
- skiphosts = list()
- bases = list()
+ results = dict()
+ hosts = dict()
+ skiphosts = dict()
+ bases = dict()
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
- self.hosts = list()
- self.skiphosts = list()
- self.bases = list()
+ self.results = dict()
+ self.hosts = dict()
+ self.skiphosts = dict()
+ self.bases = dict()
self.__dataSource__ = "Target Website"
for opt in userOpts.keys():
@@ -62,6 +62,19 @@ class sfp_junkfiles(SpiderFootPlugin):
def producedEvents(self):
return ["JUNK_FILE"]
+ # Test how trustworthy a result is
+ def checkValidity(self, junkUrl):
+ # Try and fetch an obviously missing version of the junk file
+ fetch = junkUrl + str(random.randint(0, 99999999))
+ res = self.sf.fetchUrl(fetch, headOnly=True,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
+ if res['code'] != "404":
+ host = self.sf.urlBaseUrl(junkUrl)
+ self.skiphosts[host] = True
+ return False
+ return True
+
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
@@ -74,27 +87,18 @@ class sfp_junkfiles(SpiderFootPlugin):
if eventData in self.results:
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
if self.opts['skipfake'] and host in self.skiphosts:
self.sf.debug("Skipping " + host + " because it doesn't return 404s.")
return None
- # Try and fetch an obviously missing page
- if host not in self.hosts and self.opts['skipfake']:
- fetch = host + "/" + str(random.randint(0, 99999999)) + ".html"
- res = self.sf.fetchUrl(fetch, headOnly=True,
- timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
- if res['code'] != "404":
- self.skiphosts.append(host)
- self.hosts.append(host)
- return None
-
- self.hosts.append(host)
-
# http://www/blah/abc.php -> try http://www/blah/abc.php.[fileexts]
for ext in self.opts['urlextstry']:
+ if self.opts['skipfake'] and host in self.skiphosts:
+ self.sf.debug("Skipping " + host + " because it doesn't return 404s.")
+ return None
+
if "." + ext + "?" in eventData or "." + ext + "#" in eventData or \
eventData.endswith("." + ext):
bits = eventData.split("?")
@@ -105,7 +109,7 @@ class sfp_junkfiles(SpiderFootPlugin):
self.sf.debug("Trying " + x + " against " + eventData)
fetch = bits[0] + "." + x
if fetch not in self.results:
- self.results.append(fetch)
+ self.results[fetch] = True
else:
self.sf.debug("Skipping, already fetched.")
continue
@@ -117,15 +121,17 @@ class sfp_junkfiles(SpiderFootPlugin):
self.sf.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch)
continue
if res['code'] == "200":
- evt = SpiderFootEvent("JUNK_FILE", fetch,
- self.__name__, event)
+ if not self.checkValidity(fetch):
+ continue
+
+ evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)
self.notifyListeners(evt)
base = self.sf.urlBaseDir(eventData)
if base in self.bases:
return None
else:
- self.bases.append(base)
+ self.bases[base] = True
# don't do anything with the root directory of a site
self.sf.debug("Base: " + base + ", event: " + eventData)
@@ -137,6 +143,10 @@ class sfp_junkfiles(SpiderFootPlugin):
if self.checkForStop():
return None
+ if self.opts['skipfake'] and host in self.skiphosts:
+ self.sf.debug("Skipping " + host + " because it doesn't return 404s.")
+ return None
+
if base.count('/') == 3:
self.sf.debug("Skipping base url.")
continue
@@ -144,7 +154,7 @@ class sfp_junkfiles(SpiderFootPlugin):
self.sf.debug("Trying " + dirfile + " against " + eventData)
fetch = base[0:len(base) - 1] + "." + dirfile
if fetch not in self.results:
- self.results.append(fetch)
+ self.results[fetch] = True
else:
self.sf.debug("Skipping, already fetched.")
continue
@@ -155,8 +165,10 @@ class sfp_junkfiles(SpiderFootPlugin):
self.sf.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch)
continue
if res['code'] == "200":
- evt = SpiderFootEvent("JUNK_FILE", fetch,
- self.__name__, event)
+ if not self.checkValidity(fetch):
+ continue
+
+ evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)
self.notifyListeners(evt)
# http://www/blah/abc.html -> try http://www/blah/[files]
@@ -164,10 +176,14 @@ class sfp_junkfiles(SpiderFootPlugin):
if self.checkForStop():
return None
+ if self.opts['skipfake'] and host in self.skiphosts:
+ self.sf.debug("Skipping " + host + " because it doesn't return 404s.")
+ return None
+
self.sf.debug("Trying " + f + " against " + eventData)
fetch = base + f
if fetch not in self.results:
- self.results.append(fetch)
+ self.results[fetch] = True
else:
self.sf.debug("Skipping, already fetched.")
continue
@@ -178,8 +194,10 @@ class sfp_junkfiles(SpiderFootPlugin):
self.sf.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch)
continue
if res['code'] == "200":
- evt = SpiderFootEvent("JUNK_FILE", fetch,
- self.__name__, event)
+ if not self.checkValidity(fetch):
+ continue
+
+ evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)
self.notifyListeners(evt)
return None