SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
archive.org module to identify historic pages.
Steve Micallefae3df6ff6859
diff --git a/evtypes.sql b/evtypes.sqlindex 0624efa5..82509043 100644--- a/evtypes.sql+++ b/evtypes.sql@@ -94,6 +94,14 @@ INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES (INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, "DESCRIPTOR");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, "DESCRIPTOR");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, "DESCRIPTOR");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, "DATA");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, "DATA");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, "DATA");diff --git a/modules/sfp_historic.py b/modules/sfp_historic.pynew file mode 100644index 00000000..5258cadf--- /dev/null+++ b/modules/sfp_historic.py@@ -0,0 +1,147 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_historic+# Purpose: Queries archive.org (Wayback machine) for historic versions of+# certain pages.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 20/07/2015+# Copyright: (c) Steve Micallef 2015+# Licence: GPL+# -------------------------------------------------------------------------------++import re+import urllib+import datetime+import time+import json+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+++class sfp_historic(SpiderFootPlugin):+ """Historic Files:Identifies historic versions of interesting files/pages from the Wayback Machine."""++ # Default options+ opts = {+ 'farback': "30,60,90",+ 'intfiles': True,+ 'passwordpages': True,+ 'formpages': False,+ 'flashpages': False,+ 'javapages': False,+ 'staticpages': False,+ 'uploadpages': False,+ 'webframeworkpages': False,+ 'javascriptpages': False+ }++ # Option descriptions+ optdescs = {+ 'farback': "Number of days back to look for older versions of files/pages in the Wayback Machine snapshots. Comma-separate the values, so for example 30,60,90 means to look for snapshots 30 days, 60 days and 90 days back.",+ 'intfiles': "Query the Wayback Machine for historic versions of Interesting Files.",+ 'passwordpages': "Query the Wayback Machine for historic versions of URLs with passwords.",+ 'formpages': "Query the Wayback Machine for historic versions of URLs with forms.",+ 'uploadpages': "Query the Wayback Machine for historic versions of URLs accepting uploads.",+ 'flashpages': "Query the Wayback Machine for historic versions of URLs containing Flash.",+ 'javapages': "Query the Wayback Machine for historic versions of URLs using Java Applets.",+ 'staticpages': "Query the Wayback Machine for historic versions of purely static URLs.",+ "webframeworkpages": "Query the Wayback Machine for historic versions of URLs using Javascript frameworks.",+ "javascriptpages": "Query the Wayback Machine for historic versions of URLs using Javascript."+ }++ results = list()+ foundDates = list()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = list()+ self.foundDates = list()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["INTERESTING_FILE", "URL_PASSWORD", "URL_FORM", "URL_FLASH",+ "URL_STATIC", "URL_JAVA_APPLET", "URL_UPLOAD", "URL_JAVASCRIPT",+ "URL_WEB_FRAMEWORK"]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["INTERESTING_FILE_HISTORIC", "URL_PASSWORD_HISTORIC",+ "URL_FORM_HISTORIC", "URL_FLASH_HISTORIC",+ "URL_STATIC_HISTORIC", "URL_JAVA_APPLET_HISTORIC",+ "URL_UPLOAD_HISTORIC", "URL_WEB_FRAMEWORK_HISTORIC",+ "URL_JAVASCRIPT_HISTORIC"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventName == "INTERESTING_FILE" and not self.opts['intfiles']:+ return None+ if eventName == "URL_PASSWORD" and not self.opts['passwordpages']:+ return None+ if eventName == "URL_STATIC" and not self.opts['staticpages']:+ return None+ if eventName == "URL_FORM" and not self.opts['formpages']:+ return None+ if eventName == "URL_UPLOAD" and not self.opts['uploadpages']:+ return None+ if eventName == "URL_JAVA_APPLET" and not self.opts['javapages']:+ return None+ if eventName == "URL_FLASH" and not self.opts['flashpages']:+ return None+ if eventName == "URL_JAVASCRIPT" and not self.opts['javascriptpages']:+ return None+ if eventName == "URL_WEB_FRAMEWORK" and not self.opts['webframeworkpages']:+ return None++ if eventData in self.results:+ return None+ else:+ self.results.append(eventData)++ for daysback in self.opts['farback'].split(","):+ newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))+ maxDate = newDate.strftime("%Y%m%d")++ url = "http://archive.org/wayback/available?url=" + eventData + \+ "×tamp=" + maxDate+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])++ if res == None:+ self.sf.error("Unable to fetch " + url, False)+ continue++ ret = json.loads(res['content'])+ if ret == None:+ self.sf.error("Unable to process empty response from archive.org: " + \+ eventData, False)+ continue++ if len(ret['archived_snapshots']) < 1:+ self.sf.debug("No archived snapshots for " + eventData)+ continue++ wbmlink = ret['archived_snapshots']['closest']['url']+ if wbmlink in self.foundDates:+ self.sf.debug("Snapshot already fetched.")+ continue++ self.foundDates.append(wbmlink)+ name = eventName + "_HISTORIC"++ self.sf.info("Found a historic file: " + wbmlink)+ evt = SpiderFootEvent(name, wbmlink, self.__name__, event)+ self.notifyListeners(evt)++# End of sfp_historic classdiff --git a/sfdb.py b/sfdb.pyindex 2d49afb6..e58b2038 100644--- a/sfdb.py+++ b/sfdb.py@@ -168,6 +168,14 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",