SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Correct db module.
smicallef95f5a79f9b05
diff --git a/sfdb.py b/sfdb.pyindex 454193ef..0bbb4e74 100644--- a/sfdb.py+++ b/sfdb.py@@ -1,4 +1,5 @@-#-------------------------------------------------------------------------------+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------# Name: sfdb# Purpose: Common functions for working with the database back-end.#@@ -7,16 +8,20 @@# Created: 15/05/2012# Copyright: (c) Steve Micallef 2012# Licence: GPL-#-------------------------------------------------------------------------------+# --------------------------------------------------------------------------------import hashlib-import random-import psycopg2-import sys+import sqlite3import reimport timefrom sflib import SpiderFoot++def __dbregex__(qry, data):+ rx = re.compile(qry, re.IGNORECASE)+ ret = rx.search(data)+ return ret is not None++class SpiderFootDb:sf = Nonedbh = None@@ -24,146 +29,147 @@ class SpiderFootDb:# Queries for creating the SpiderFoot databasecreateQueries = [- "CREATE TABLE tbl_event_types ( \- event VARCHAR NOT NULL PRIMARY KEY, \- event_descr VARCHAR NOT NULL, \- event_raw INT NOT NULL DEFAULT 0, \- event_type VARCHAR NOT NULL \- )",- "CREATE TABLE tbl_config ( \- scope VARCHAR NOT NULL, \- opt VARCHAR NOT NULL, \- val VARCHAR NOT NULL, \- PRIMARY KEY (scope, opt) \- )",- "CREATE TABLE tbl_scan_instance ( \- guid VARCHAR NOT NULL PRIMARY KEY, \- name VARCHAR NOT NULL, \- seed_target VARCHAR NOT NULL, \- created BIGINT DEFAULT 0, \- started BIGINT DEFAULT 0, \- ended BIGINT DEFAULT 0, \- status VARCHAR NOT NULL \- )",- "CREATE TABLE tbl_scan_log ( \- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \- generated BIGINT NOT NULL, \- component VARCHAR, \- type VARCHAR NOT NULL, \- message TEXT \- )",- "CREATE TABLE tbl_scan_config ( \- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \- component VARCHAR NOT NULL, \- opt VARCHAR NOT NULL, \- val VARCHAR NOT NULL \- )",- "CREATE TABLE tbl_scan_results ( \- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \- hash VARCHAR NOT NULL, \- type VARCHAR NOT NULL REFERENCES tbl_event_types(event), \- generated BIGINT NOT NULL, \- confidence INT NOT NULL DEFAULT 100, \- visibility INT NOT NULL DEFAULT 100, \- risk INT NOT NULL DEFAULT 0, \- module VARCHAR NOT NULL, \- data TEXT, \- source_event_hash VARCHAR DEFAULT 'ROOT' \- )",- "CREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)",- "CREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)",- "CREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)",- "CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)",- "CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PASTEBIN_CONTENT', 'PasteBin Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')"+ "PRAGMA journal_mode=WAL",+ "CREATE TABLE tbl_event_types ( \+ event VARCHAR NOT NULL PRIMARY KEY, \+ event_descr VARCHAR NOT NULL, \+ event_raw INT NOT NULL DEFAULT 0, \+ event_type VARCHAR NOT NULL \+ )",+ "CREATE TABLE tbl_config ( \+ scope VARCHAR NOT NULL, \+ opt VARCHAR NOT NULL, \+ val VARCHAR NOT NULL, \+ PRIMARY KEY (scope, opt) \+ )",+ "CREATE TABLE tbl_scan_instance ( \+ guid VARCHAR NOT NULL PRIMARY KEY, \+ name VARCHAR NOT NULL, \+ seed_target VARCHAR NOT NULL, \+ created INT DEFAULT 0, \+ started INT DEFAULT 0, \+ ended INT DEFAULT 0, \+ status VARCHAR NOT NULL \+ )",+ "CREATE TABLE tbl_scan_log ( \+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \+ generated INT NOT NULL, \+ component VARCHAR, \+ type VARCHAR NOT NULL, \+ message VARCHAR \+ )",+ "CREATE TABLE tbl_scan_config ( \+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \+ component VARCHAR NOT NULL, \+ opt VARCHAR NOT NULL, \+ val VARCHAR NOT NULL \+ )",+ "CREATE TABLE tbl_scan_results ( \+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \+ hash VARCHAR NOT NULL, \+ type VARCHAR NOT NULL REFERENCES tbl_event_types(event), \+ generated INT NOT NULL, \+ confidence INT NOT NULL DEFAULT 100, \+ visibility INT NOT NULL DEFAULT 100, \+ risk INT NOT NULL DEFAULT 0, \+ module VARCHAR NOT NULL, \+ data VARCHAR, \+ source_event_hash VARCHAR DEFAULT 'ROOT' \+ )",+ "CREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)",+ "CREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)",+ "CREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)",+ "CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)",+ "CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PASTEBIN_CONTENT', 'PasteBin Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')"]def __init__(self, opts):@@ -172,26 +178,25 @@ class SpiderFootDb:# connect() will create the database file if it doesn't exist, but# at least we can use this opportunity to ensure we have permissions to# read and write to such a file.- dbh = psycopg2.connect(database=opts['__pgdb'], user=opts['__pguser'],- password=opts['__pgpass'], host=opts['__pghost'], port=opts['__pgport'])- if dbh == None:- self.sf.fatal("Could not connect to PostgreSQL database")+ dbh = sqlite3.connect(self.sf.myPath() + "/" + opts['__database'], timeout=10)+ if dbh is None:+ self.sf.fatal("Could not connect to internal database, and couldn't create " + opts['__database'])+ dbh.text_factory = strself.conn = dbh- self.conn.set_client_encoding("utf-8")self.dbh = dbh.cursor()# Now we actually check to ensure the database file has the schema set# up correctly.try:self.dbh.execute('SELECT COUNT(*) FROM tbl_scan_config')- except psycopg2.Error:+ self.conn.create_function("REGEXP", 2, __dbregex__)+ except sqlite3.Error:# .. If not set up, we set it up.try:self.create()except BaseException as e:- self.sf.error("Tried to set up the SpiderFoot database schema, but failed: " + \- e.args[0])+ self.sf.error("Tried to set up the SpiderFoot database schema, but failed: " + e.args[0])return#@@ -204,9 +209,8 @@ class SpiderFootDb:for qry in self.createQueries:self.dbh.execute(qry)self.conn.commit()- except psycopg2.Error as e:- raise BaseException("SQL error encountered when setting up database: " +- e.args[0])+ except sqlite3.Error as e:+ raise BaseException("SQL error encountered when setting up database: " + e.args[0])# Close the database handledef close(self):@@ -225,37 +229,36 @@ class SpiderFootDb:qvars = list()qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as source_data, \+ s.data as 'source_data', \c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \c.source_event_hash, t.event_descr, c.scan_instance_id \FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \WHERE s.scan_instance_id = c.scan_instance_id AND \t.event = c.type AND c.source_event_hash = s.hash "- if criteria.get('scan_id') != None:- qry += "AND c.scan_instance_id = %s "+ if criteria.get('scan_id') is not None:+ qry += "AND c.scan_instance_id = ? "qvars.append(criteria['scan_id'])- if criteria.get('type') != None:- qry += " AND c.type = %s "+ if criteria.get('type') is not None:+ qry += " AND c.type = ? "qvars.append(criteria['type'])- if criteria.get('value') != None:- qry += " AND c.data LIKE %s "+ if criteria.get('value') is not None:+ qry += " AND c.data LIKE ? "qvars.append(criteria['value'])- if criteria.get('regex') != None:- qry += " AND c.data ~* %s "+ if criteria.get('regex') is not None:+ qry += " AND c.data REGEXP ? "qvars.append(criteria['regex'])- qry = qry + " ORDER BY c.data"+ qry += " ORDER BY c.data"try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching search results: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching search results: " + e.args[0])# Get event typesdef eventTypes(self):@@ -263,27 +266,26 @@ class SpiderFootDb:try:self.dbh.execute(qry)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when retreiving event types:" +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when retreiving event types:" + e.args[0])# Log an event to the databasedef scanLogEvent(self, instanceId, classification, message, component=None):- if component == None:+ if component is None:component = "SpiderFoot"qry = "INSERT INTO tbl_scan_log \(scan_instance_id, generated, component, type, message) \- VALUES (%s, %s, %s, %s, %s)"+ VALUES (?, ?, ?, ?, ?)"try:self.dbh.execute(qry, (- instanceId, time.time() * 1000, component, classification, message- ))+ instanceId, time.time() * 1000, component, classification, message+ ))self.conn.commit()- except psycopg2.Error as e:+ except sqlite3.Error as e:if "locked" in e.args[0]:# TODO: Do something smarter here to handle locked databases- self.sf.fatal("Unable to log event in DB: " + e.args[0])+ self.sf.fatal("Unable to log event in DB due to lock: " + e.args[0])else:self.sf.fatal("Unable to log event in DB: " + e.args[0])@@ -293,13 +295,13 @@ class SpiderFootDb:def scanInstanceCreate(self, instanceId, scanName, scanTarget):qry = "INSERT INTO tbl_scan_instance \(guid, name, seed_target, created, status) \- VALUES (%s, %s, %s, %s, %s)"+ VALUES (?, ?, ?, ?, ?)"try:self.dbh.execute(qry, (- instanceId, scanName, scanTarget, time.time() * 1000, 'CREATED'- ))+ instanceId, scanName, scanTarget, time.time() * 1000, 'CREATED'+ ))self.conn.commit()- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.fatal("Unable to create instance in DB: " + e.args[0])return True@@ -309,26 +311,26 @@ class SpiderFootDb:qvars = list()qry = "UPDATE tbl_scan_instance SET "- if started != None:- qry += " started = %s,"+ if started is not None:+ qry += " started = ?,"qvars.append(started)- if ended != None:- qry += " ended = %s,"+ if ended is not None:+ qry += " ended = ?,"qvars.append(ended)- if status != None:- qry += " status = %s,"+ if status is not None:+ qry += " status = ?,"qvars.append(status)# guid = guid is a little hack to avoid messing with , placement above- qry += " guid = guid WHERE guid = %s"+ qry += " guid = guid WHERE guid = ?"qvars.append(instanceId)try:self.dbh.execute(qry, qvars)self.conn.commit()- except psycopg2.Error:+ except sqlite3.Error:self.sf.fatal("Unable to set information for the scan instance.")# Return info about a scan instance (name, target, created, started,@@ -336,37 +338,35 @@ class SpiderFootDb:def scanInstanceGet(self, instanceId):qry = "SELECT name, seed_target, ROUND(created/1000) AS created, \ROUND(started/1000) AS started, ROUND(ended/1000) AS ended, status \- FROM tbl_scan_instance WHERE guid = %s"+ FROM tbl_scan_instance WHERE guid = ?"qvars = [instanceId]try:self.dbh.execute(qry, qvars)return self.dbh.fetchone()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when retreiving scan instance:" +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when retreiving scan instance:" + e.args[0])# Obtain a summary of the results per event typedef scanResultSummary(self, instanceId):qry = "SELECT r.type, e.event_descr, MAX(ROUND(generated)) AS last_in, \count(*) AS total, count(DISTINCT r.data) as utotal FROM \tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \- AND r.scan_instance_id = %s GROUP BY r.type, e.event_descr ORDER BY e.event_descr"+ AND r.scan_instance_id = ? GROUP BY r.type ORDER BY e.event_descr"qvars = [instanceId]try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching result summary: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching result summary: " + e.args[0])# Obtain the ROOT event for a scan: Must be same output as scanResultEvent!def scanRootEvent(self, instanceId):qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as source_data, \+ s.data as 'source_data', \c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id \FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE c.scan_instance_id = %s AND c.source_event_hash = s.hash AND \+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \s.scan_instance_id = c.scan_instance_id AND \t.event = c.type AND c.source_event_hash = 'ROOT'"@@ -375,97 +375,92 @@ class SpiderFootDb:try:self.dbh.execute(qry, qvars)return self.dbh.fetchone()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching ROOT event: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching ROOT event: " + e.args[0])# Obtain the data for a scan and event typedef scanResultEvent(self, instanceId, eventType='ALL'):qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data AS source_data, \+ s.data as 'source_data', \c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id \FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE c.scan_instance_id = %s AND c.source_event_hash = s.hash AND \+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \s.scan_instance_id = c.scan_instance_id AND \t.event = c.type"qvars = [instanceId]if eventType != "ALL":- qry = qry + " AND c.type = %s"+ qry += " AND c.type = ?"qvars.append(eventType)- qry = qry + " ORDER BY c.data"+ qry += " ORDER BY c.data"try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching result events: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching result events: " + e.args[0])# Obtain a unique list of elementsdef scanResultEventUnique(self, instanceId, eventType='ALL'):qry = "SELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results \- WHERE scan_instance_id = %s"+ WHERE scan_instance_id = ?"qvars = [instanceId]if eventType != "ALL":- qry = qry + " AND type = %s"+ qry += " AND type = ?"qvars.append(eventType)- qry = qry + " GROUP BY type, data ORDER BY COUNT(*)"+ qry += " GROUP BY type, data ORDER BY COUNT(*)"try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching unique result events: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching unique result events: " + e.args[0])# Get scan logsdef scanLogs(self, instanceId, limit=None):qry = "SELECT generated AS generated, component, \- type, message FROM tbl_scan_log WHERE scan_instance_id = %s \+ type, message FROM tbl_scan_log WHERE scan_instance_id = ? \ORDER BY generated DESC"qvars = [instanceId]- if limit != None:- qry = qry + " LIMIT %s"+ if limit is not None:+ qry += " LIMIT ?"qvars.append(limit)try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching scan logs: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching scan logs: " + e.args[0])# Get scan errorsdef scanErrors(self, instanceId, limit=None):qry = "SELECT generated AS generated, component, \- message FROM tbl_scan_log WHERE scan_instance_id = %s \+ message FROM tbl_scan_log WHERE scan_instance_id = ? \AND type = 'ERROR' ORDER BY generated DESC"qvars = [instanceId]- if limit != None:- qry = qry + " LIMIT %s"+ if limit is not None:+ qry += " LIMIT ?"qvars.append(limit)try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when fetching scan errors: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching scan errors: " + e.args[0])# Delete a scan instancedef scanInstanceDelete(self, instanceId):- qry4 = "DELETE FROM tbl_scan_instance WHERE guid = %s"- qry1 = "DELETE FROM tbl_scan_config WHERE scan_instance_id = %s"- qry3 = "DELETE FROM tbl_scan_results WHERE scan_instance_id = %s"- qry2 = "DELETE FROM tbl_scan_log WHERE scan_instance_id = %s"+ qry1 = "DELETE FROM tbl_scan_instance WHERE guid = ?"+ qry2 = "DELETE FROM tbl_scan_config WHERE scan_instance_id = ?"+ qry3 = "DELETE FROM tbl_scan_results WHERE scan_instance_id = ?"+ qry4 = "DELETE FROM tbl_scan_log WHERE scan_instance_id = ?"qvars = [instanceId]try:self.dbh.execute(qry1, qvars)@@ -473,33 +468,25 @@ class SpiderFootDb:self.dbh.execute(qry3, qvars)self.dbh.execute(qry4, qvars)self.conn.commit()- except psycopg2.Error as e:- self.sf.error("SQL error encountered when deleting scan: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when deleting scan: " + e.args[0])# Store the default configurationdef configSet(self, optMap=dict()):+ qry = "REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)"for opt in optMap.keys():- qry = "INSERT INTO tbl_config (scope, opt, val) VALUES (%s, %s, %s)"# Module optionif ":" in opt:parts = opt.split(':')- qvals = [ parts[0], parts[1], optMap[opt] ]+ qvals = [parts[0], parts[1], optMap[opt]]else:- # Global option- qvals = [ "GLOBAL", opt, optMap[opt] ]+ # Global option+ qvals = ["GLOBAL", opt, optMap[opt]]try:self.dbh.execute(qry, qvals)- except psycopg2.Error as e:- self.conn.rollback()- try:- qry = "UPDATE tbl_config SET val = %s WHERE scope = %s AND opt = %s"- nvals = [qvals[2], qvals[0], qvals[1] ]- self.dbh.execute(qry, nvals)- except psycopg2.Error as e:- self.sf.error("SQL error encountered when storing config, aborting: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])self.conn.commit()@@ -516,7 +503,7 @@ class SpiderFootDb:retval[scope + ":" + opt] = valreturn retval- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])# Reset the config to default (clear it from the DB and let the hard-coded@@ -526,44 +513,34 @@ class SpiderFootDb:try:self.dbh.execute(qry)self.conn.commit()- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.error("Unable to clear configuration from the database: " + e.args[0])# Store a configuration value for a scandef scanConfigSet(self, id, optMap=dict()):- for opt in optMap.keys():- qry = "INSERT INTO tbl_scan_config \- (scan_instance_id, component, opt, val) VALUES (%s, %s, %s, %s)"+ qry = "REPLACE INTO tbl_scan_config \+ (scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)"+ for opt in optMap.keys():# Module optionif ":" in opt:parts = opt.split(':')- qvals = [ id, parts[0], parts[1], optMap[opt] ]+ qvals = [id, parts[0], parts[1], optMap[opt]]else:- # Global option- qvals = [ id, "GLOBAL", opt, optMap[opt] ]+ # Global option+ qvals = [id, "GLOBAL", opt, optMap[opt]]try:self.dbh.execute(qry, qvals)- except psycopg2.Error as e:- # Try updating if inserting failed- self.conn.rollback()- try:- qry = "UPDATE tbl_scan_config \- SET val = %s WHERE scan_instance_id = %s AND \- component = %s AND opt = %s"- nvals = [ qvals[3], qvals[0], qvals[1], qvals[2] ]- self.dbh.execute(qry, nvals)- except psycogp2.Error as e:- self.sf.error("SQL error encountered when storing config, aborting: " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])self.conn.commit()# Retreive configuration data for a scan componentdef scanConfigGet(self, instanceId):qry = "SELECT component, opt, val FROM tbl_scan_config \- WHERE scan_instance_id = %s ORDER BY component, opt"+ WHERE scan_instance_id = ? ORDER BY component, opt"qvars = [instanceId]try:retval = dict()@@ -574,7 +551,7 @@ class SpiderFootDb:else:retval[component + ":" + opt] = valreturn retval- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])# Store an event@@ -607,16 +584,16 @@ class SpiderFootDb:if truncateSize > 0:storeData = storeData[0:truncateSize]- if sfEvent.sourceEventHash in [ "", None]:+ if sfEvent.sourceEventHash in ["", None]:self.sf.fatal("UNABLE TO CREATE RECORD WITH EMPTY SOURCE EVENT HASH!")qry = "INSERT INTO tbl_scan_results \(scan_instance_id, hash, type, generated, confidence, \visibility, risk, module, data, source_event_hash) \- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"- qvals = [ instanceId, sfEvent.getHash(), sfEvent.eventType, sfEvent.generated,- sfEvent.confidence, sfEvent.visibility, sfEvent.risk,- sfEvent.module, storeData, sfEvent.sourceEventHash ]+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"+ qvals = [instanceId, sfEvent.getHash(), sfEvent.eventType, sfEvent.generated,+ sfEvent.confidence, sfEvent.visibility, sfEvent.risk,+ sfEvent.module, storeData, sfEvent.sourceEventHash]#print "STORING: " + str(qvals)@@ -624,9 +601,8 @@ class SpiderFootDb:self.dbh.execute(qry, qvals)self.conn.commit()return None- except psycopg2.Error as e:- self.sf.fatal("SQL error encountered when storing event data (" + str(self.dbh) + ": " +- e.args[0])+ except sqlite3.Error as e:+ self.sf.fatal("SQL error encountered when storing event data (" + str(self.dbh) + ": " + e.args[0])# List of all previously run scansdef scanInstanceList(self):@@ -646,19 +622,19 @@ class SpiderFootDb:try:self.dbh.execute(qry)return self.dbh.fetchall()- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.error("SQL error encountered when fetching scan list: " + e.args[0])# History of data from the scandef scanResultHistory(self, instanceId):qry = "SELECT STRFTIME('%H:%M %w', generated, 'unixepoch') AS hourmin, \type, COUNT(*) FROM tbl_scan_results \- WHERE scan_instance_id = %s GROUP BY hourmin, type"+ WHERE scan_instance_id = ? GROUP BY hourmin, type"qvars = [instanceId]try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.error("SQL error encountered when fetching scan history: " + e.args[0])@@ -667,23 +643,21 @@ class SpiderFootDb:# the output of this needs to be aligned with scanResultEvent,# as other functions call both expecting the same output.qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as source_data, \+ s.data as 'source_data', \c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \c.source_event_hash, t.event_descr \FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE c.scan_instance_id = %s AND c.source_event_hash = s.hash AND \+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \s.scan_instance_id = c.scan_instance_id AND \t.event = c.type AND c.hash in ("qvars = [instanceId]for hashId in elementIdList:qry = qry + "'" + hashId + "',"- qry = qry + "'')"+ qry += "'')"try:self.dbh.execute(qry, qvars)return self.dbh.fetchall()- except psycopg2.Error as e:+ except sqlite3.Error as e:self.sf.error("SQL error encountered when getting source element IDs: " + e.args[0])--