Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Correct db module.

smicallef95f5a79f9b05
diff --git a/sfdb.py b/sfdb.py
index 454193ef..0bbb4e74 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -1,4 +1,5 @@
-#-------------------------------------------------------------------------------
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
# Name: sfdb
# Purpose: Common functions for working with the database back-end.
#
@@ -7,16 +8,20 @@
# Created: 15/05/2012
# Copyright: (c) Steve Micallef 2012
# Licence: GPL
-#-------------------------------------------------------------------------------
+# -------------------------------------------------------------------------------
-import hashlib
-import random
-import psycopg2
-import sys
+import sqlite3
import re
import time
from sflib import SpiderFoot
+
+def __dbregex__(qry, data):
+ rx = re.compile(qry, re.IGNORECASE)
+ ret = rx.search(data)
+ return ret is not None
+
+
class SpiderFootDb:
sf = None
dbh = None
@@ -24,146 +29,147 @@ class SpiderFootDb:
# Queries for creating the SpiderFoot database
createQueries = [
- "CREATE TABLE tbl_event_types ( \
- event VARCHAR NOT NULL PRIMARY KEY, \
- event_descr VARCHAR NOT NULL, \
- event_raw INT NOT NULL DEFAULT 0, \
- event_type VARCHAR NOT NULL \
- )",
- "CREATE TABLE tbl_config ( \
- scope VARCHAR NOT NULL, \
- opt VARCHAR NOT NULL, \
- val VARCHAR NOT NULL, \
- PRIMARY KEY (scope, opt) \
- )",
- "CREATE TABLE tbl_scan_instance ( \
- guid VARCHAR NOT NULL PRIMARY KEY, \
- name VARCHAR NOT NULL, \
- seed_target VARCHAR NOT NULL, \
- created BIGINT DEFAULT 0, \
- started BIGINT DEFAULT 0, \
- ended BIGINT DEFAULT 0, \
- status VARCHAR NOT NULL \
- )",
- "CREATE TABLE tbl_scan_log ( \
- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \
- generated BIGINT NOT NULL, \
- component VARCHAR, \
- type VARCHAR NOT NULL, \
- message TEXT \
- )",
- "CREATE TABLE tbl_scan_config ( \
- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \
- component VARCHAR NOT NULL, \
- opt VARCHAR NOT NULL, \
- val VARCHAR NOT NULL \
- )",
- "CREATE TABLE tbl_scan_results ( \
- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \
- hash VARCHAR NOT NULL, \
- type VARCHAR NOT NULL REFERENCES tbl_event_types(event), \
- generated BIGINT NOT NULL, \
- confidence INT NOT NULL DEFAULT 100, \
- visibility INT NOT NULL DEFAULT 100, \
- risk INT NOT NULL DEFAULT 0, \
- module VARCHAR NOT NULL, \
- data TEXT, \
- source_event_hash VARCHAR DEFAULT 'ROOT' \
- )",
- "CREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)",
- "CREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)",
- "CREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)",
- "CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)",
- "CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PASTEBIN_CONTENT', 'PasteBin Content', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')"
+ "PRAGMA journal_mode=WAL",
+ "CREATE TABLE tbl_event_types ( \
+ event VARCHAR NOT NULL PRIMARY KEY, \
+ event_descr VARCHAR NOT NULL, \
+ event_raw INT NOT NULL DEFAULT 0, \
+ event_type VARCHAR NOT NULL \
+ )",
+ "CREATE TABLE tbl_config ( \
+ scope VARCHAR NOT NULL, \
+ opt VARCHAR NOT NULL, \
+ val VARCHAR NOT NULL, \
+ PRIMARY KEY (scope, opt) \
+ )",
+ "CREATE TABLE tbl_scan_instance ( \
+ guid VARCHAR NOT NULL PRIMARY KEY, \
+ name VARCHAR NOT NULL, \
+ seed_target VARCHAR NOT NULL, \
+ created INT DEFAULT 0, \
+ started INT DEFAULT 0, \
+ ended INT DEFAULT 0, \
+ status VARCHAR NOT NULL \
+ )",
+ "CREATE TABLE tbl_scan_log ( \
+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \
+ generated INT NOT NULL, \
+ component VARCHAR, \
+ type VARCHAR NOT NULL, \
+ message VARCHAR \
+ )",
+ "CREATE TABLE tbl_scan_config ( \
+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \
+ component VARCHAR NOT NULL, \
+ opt VARCHAR NOT NULL, \
+ val VARCHAR NOT NULL \
+ )",
+ "CREATE TABLE tbl_scan_results ( \
+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \
+ hash VARCHAR NOT NULL, \
+ type VARCHAR NOT NULL REFERENCES tbl_event_types(event), \
+ generated INT NOT NULL, \
+ confidence INT NOT NULL DEFAULT 100, \
+ visibility INT NOT NULL DEFAULT 100, \
+ risk INT NOT NULL DEFAULT 0, \
+ module VARCHAR NOT NULL, \
+ data VARCHAR, \
+ source_event_hash VARCHAR DEFAULT 'ROOT' \
+ )",
+ "CREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)",
+ "CREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)",
+ "CREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)",
+ "CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)",
+ "CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PASTEBIN_CONTENT', 'PasteBin Content', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')"
]
def __init__(self, opts):
@@ -172,26 +178,25 @@ class SpiderFootDb:
# connect() will create the database file if it doesn't exist, but
# at least we can use this opportunity to ensure we have permissions to
# read and write to such a file.
- dbh = psycopg2.connect(database=opts['__pgdb'], user=opts['__pguser'],
- password=opts['__pgpass'], host=opts['__pghost'], port=opts['__pgport'])
- if dbh == None:
- self.sf.fatal("Could not connect to PostgreSQL database")
+ dbh = sqlite3.connect(self.sf.myPath() + "/" + opts['__database'], timeout=10)
+ if dbh is None:
+ self.sf.fatal("Could not connect to internal database, and couldn't create " + opts['__database'])
+ dbh.text_factory = str
self.conn = dbh
- self.conn.set_client_encoding("utf-8")
self.dbh = dbh.cursor()
# Now we actually check to ensure the database file has the schema set
# up correctly.
try:
self.dbh.execute('SELECT COUNT(*) FROM tbl_scan_config')
- except psycopg2.Error:
+ self.conn.create_function("REGEXP", 2, __dbregex__)
+ except sqlite3.Error:
# .. If not set up, we set it up.
try:
self.create()
except BaseException as e:
- self.sf.error("Tried to set up the SpiderFoot database schema, but failed: " + \
- e.args[0])
+ self.sf.error("Tried to set up the SpiderFoot database schema, but failed: " + e.args[0])
return
#
@@ -204,9 +209,8 @@ class SpiderFootDb:
for qry in self.createQueries:
self.dbh.execute(qry)
self.conn.commit()
- except psycopg2.Error as e:
- raise BaseException("SQL error encountered when setting up database: " +
- e.args[0])
+ except sqlite3.Error as e:
+ raise BaseException("SQL error encountered when setting up database: " + e.args[0])
# Close the database handle
def close(self):
@@ -225,37 +229,36 @@ class SpiderFootDb:
qvars = list()
qry = "SELECT ROUND(c.generated) AS generated, c.data, \
- s.data as source_data, \
+ s.data as 'source_data', \
c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \
c.source_event_hash, t.event_descr, c.scan_instance_id \
FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \
WHERE s.scan_instance_id = c.scan_instance_id AND \
t.event = c.type AND c.source_event_hash = s.hash "
- if criteria.get('scan_id') != None:
- qry += "AND c.scan_instance_id = %s "
+ if criteria.get('scan_id') is not None:
+ qry += "AND c.scan_instance_id = ? "
qvars.append(criteria['scan_id'])
- if criteria.get('type') != None:
- qry += " AND c.type = %s "
+ if criteria.get('type') is not None:
+ qry += " AND c.type = ? "
qvars.append(criteria['type'])
- if criteria.get('value') != None:
- qry += " AND c.data LIKE %s "
+ if criteria.get('value') is not None:
+ qry += " AND c.data LIKE ? "
qvars.append(criteria['value'])
- if criteria.get('regex') != None:
- qry += " AND c.data ~* %s "
+ if criteria.get('regex') is not None:
+ qry += " AND c.data REGEXP ? "
qvars.append(criteria['regex'])
- qry = qry + " ORDER BY c.data"
+ qry += " ORDER BY c.data"
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching search results: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching search results: " + e.args[0])
# Get event types
def eventTypes(self):
@@ -263,27 +266,26 @@ class SpiderFootDb:
try:
self.dbh.execute(qry)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when retreiving event types:" +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when retreiving event types:" + e.args[0])
# Log an event to the database
def scanLogEvent(self, instanceId, classification, message, component=None):
- if component == None:
+ if component is None:
component = "SpiderFoot"
qry = "INSERT INTO tbl_scan_log \
(scan_instance_id, generated, component, type, message) \
- VALUES (%s, %s, %s, %s, %s)"
+ VALUES (?, ?, ?, ?, ?)"
try:
self.dbh.execute(qry, (
- instanceId, time.time() * 1000, component, classification, message
- ))
+ instanceId, time.time() * 1000, component, classification, message
+ ))
self.conn.commit()
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
if "locked" in e.args[0]:
# TODO: Do something smarter here to handle locked databases
- self.sf.fatal("Unable to log event in DB: " + e.args[0])
+ self.sf.fatal("Unable to log event in DB due to lock: " + e.args[0])
else:
self.sf.fatal("Unable to log event in DB: " + e.args[0])
@@ -293,13 +295,13 @@ class SpiderFootDb:
def scanInstanceCreate(self, instanceId, scanName, scanTarget):
qry = "INSERT INTO tbl_scan_instance \
(guid, name, seed_target, created, status) \
- VALUES (%s, %s, %s, %s, %s)"
+ VALUES (?, ?, ?, ?, ?)"
try:
self.dbh.execute(qry, (
- instanceId, scanName, scanTarget, time.time() * 1000, 'CREATED'
- ))
+ instanceId, scanName, scanTarget, time.time() * 1000, 'CREATED'
+ ))
self.conn.commit()
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.fatal("Unable to create instance in DB: " + e.args[0])
return True
@@ -309,26 +311,26 @@ class SpiderFootDb:
qvars = list()
qry = "UPDATE tbl_scan_instance SET "
- if started != None:
- qry += " started = %s,"
+ if started is not None:
+ qry += " started = ?,"
qvars.append(started)
- if ended != None:
- qry += " ended = %s,"
+ if ended is not None:
+ qry += " ended = ?,"
qvars.append(ended)
- if status != None:
- qry += " status = %s,"
+ if status is not None:
+ qry += " status = ?,"
qvars.append(status)
# guid = guid is a little hack to avoid messing with , placement above
- qry += " guid = guid WHERE guid = %s"
+ qry += " guid = guid WHERE guid = ?"
qvars.append(instanceId)
try:
self.dbh.execute(qry, qvars)
self.conn.commit()
- except psycopg2.Error:
+ except sqlite3.Error:
self.sf.fatal("Unable to set information for the scan instance.")
# Return info about a scan instance (name, target, created, started,
@@ -336,37 +338,35 @@ class SpiderFootDb:
def scanInstanceGet(self, instanceId):
qry = "SELECT name, seed_target, ROUND(created/1000) AS created, \
ROUND(started/1000) AS started, ROUND(ended/1000) AS ended, status \
- FROM tbl_scan_instance WHERE guid = %s"
+ FROM tbl_scan_instance WHERE guid = ?"
qvars = [instanceId]
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchone()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when retreiving scan instance:" +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when retreiving scan instance:" + e.args[0])
# Obtain a summary of the results per event type
def scanResultSummary(self, instanceId):
qry = "SELECT r.type, e.event_descr, MAX(ROUND(generated)) AS last_in, \
count(*) AS total, count(DISTINCT r.data) as utotal FROM \
tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \
- AND r.scan_instance_id = %s GROUP BY r.type, e.event_descr ORDER BY e.event_descr"
+ AND r.scan_instance_id = ? GROUP BY r.type ORDER BY e.event_descr"
qvars = [instanceId]
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching result summary: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching result summary: " + e.args[0])
# Obtain the ROOT event for a scan: Must be same output as scanResultEvent!
def scanRootEvent(self, instanceId):
qry = "SELECT ROUND(c.generated) AS generated, c.data, \
- s.data as source_data, \
+ s.data as 'source_data', \
c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \
c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id \
FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \
- WHERE c.scan_instance_id = %s AND c.source_event_hash = s.hash AND \
+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \
s.scan_instance_id = c.scan_instance_id AND \
t.event = c.type AND c.source_event_hash = 'ROOT'"
@@ -375,97 +375,92 @@ class SpiderFootDb:
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchone()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching ROOT event: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching ROOT event: " + e.args[0])
# Obtain the data for a scan and event type
def scanResultEvent(self, instanceId, eventType='ALL'):
qry = "SELECT ROUND(c.generated) AS generated, c.data, \
- s.data AS source_data, \
+ s.data as 'source_data', \
c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \
c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id \
FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \
- WHERE c.scan_instance_id = %s AND c.source_event_hash = s.hash AND \
+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \
s.scan_instance_id = c.scan_instance_id AND \
t.event = c.type"
qvars = [instanceId]
if eventType != "ALL":
- qry = qry + " AND c.type = %s"
+ qry += " AND c.type = ?"
qvars.append(eventType)
- qry = qry + " ORDER BY c.data"
+ qry += " ORDER BY c.data"
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching result events: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching result events: " + e.args[0])
# Obtain a unique list of elements
def scanResultEventUnique(self, instanceId, eventType='ALL'):
qry = "SELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results \
- WHERE scan_instance_id = %s"
+ WHERE scan_instance_id = ?"
qvars = [instanceId]
if eventType != "ALL":
- qry = qry + " AND type = %s"
+ qry += " AND type = ?"
qvars.append(eventType)
- qry = qry + " GROUP BY type, data ORDER BY COUNT(*)"
+ qry += " GROUP BY type, data ORDER BY COUNT(*)"
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching unique result events: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching unique result events: " + e.args[0])
# Get scan logs
def scanLogs(self, instanceId, limit=None):
qry = "SELECT generated AS generated, component, \
- type, message FROM tbl_scan_log WHERE scan_instance_id = %s \
+ type, message FROM tbl_scan_log WHERE scan_instance_id = ? \
ORDER BY generated DESC"
qvars = [instanceId]
- if limit != None:
- qry = qry + " LIMIT %s"
+ if limit is not None:
+ qry += " LIMIT ?"
qvars.append(limit)
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching scan logs: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching scan logs: " + e.args[0])
# Get scan errors
def scanErrors(self, instanceId, limit=None):
qry = "SELECT generated AS generated, component, \
- message FROM tbl_scan_log WHERE scan_instance_id = %s \
+ message FROM tbl_scan_log WHERE scan_instance_id = ? \
AND type = 'ERROR' ORDER BY generated DESC"
qvars = [instanceId]
- if limit != None:
- qry = qry + " LIMIT %s"
+ if limit is not None:
+ qry += " LIMIT ?"
qvars.append(limit)
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when fetching scan errors: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when fetching scan errors: " + e.args[0])
# Delete a scan instance
def scanInstanceDelete(self, instanceId):
- qry4 = "DELETE FROM tbl_scan_instance WHERE guid = %s"
- qry1 = "DELETE FROM tbl_scan_config WHERE scan_instance_id = %s"
- qry3 = "DELETE FROM tbl_scan_results WHERE scan_instance_id = %s"
- qry2 = "DELETE FROM tbl_scan_log WHERE scan_instance_id = %s"
+ qry1 = "DELETE FROM tbl_scan_instance WHERE guid = ?"
+ qry2 = "DELETE FROM tbl_scan_config WHERE scan_instance_id = ?"
+ qry3 = "DELETE FROM tbl_scan_results WHERE scan_instance_id = ?"
+ qry4 = "DELETE FROM tbl_scan_log WHERE scan_instance_id = ?"
qvars = [instanceId]
try:
self.dbh.execute(qry1, qvars)
@@ -473,33 +468,25 @@ class SpiderFootDb:
self.dbh.execute(qry3, qvars)
self.dbh.execute(qry4, qvars)
self.conn.commit()
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when deleting scan: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when deleting scan: " + e.args[0])
# Store the default configuration
def configSet(self, optMap=dict()):
+ qry = "REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)"
for opt in optMap.keys():
- qry = "INSERT INTO tbl_config (scope, opt, val) VALUES (%s, %s, %s)"
# Module option
if ":" in opt:
parts = opt.split(':')
- qvals = [ parts[0], parts[1], optMap[opt] ]
+ qvals = [parts[0], parts[1], optMap[opt]]
else:
- # Global option
- qvals = [ "GLOBAL", opt, optMap[opt] ]
+ # Global option
+ qvals = ["GLOBAL", opt, optMap[opt]]
try:
self.dbh.execute(qry, qvals)
- except psycopg2.Error as e:
- self.conn.rollback()
- try:
- qry = "UPDATE tbl_config SET val = %s WHERE scope = %s AND opt = %s"
- nvals = [qvals[2], qvals[0], qvals[1] ]
- self.dbh.execute(qry, nvals)
- except psycopg2.Error as e:
- self.sf.error("SQL error encountered when storing config, aborting: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])
self.conn.commit()
@@ -516,7 +503,7 @@ class SpiderFootDb:
retval[scope + ":" + opt] = val
return retval
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])
# Reset the config to default (clear it from the DB and let the hard-coded
@@ -526,44 +513,34 @@ class SpiderFootDb:
try:
self.dbh.execute(qry)
self.conn.commit()
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.error("Unable to clear configuration from the database: " + e.args[0])
# Store a configuration value for a scan
def scanConfigSet(self, id, optMap=dict()):
- for opt in optMap.keys():
- qry = "INSERT INTO tbl_scan_config \
- (scan_instance_id, component, opt, val) VALUES (%s, %s, %s, %s)"
+ qry = "REPLACE INTO tbl_scan_config \
+ (scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)"
+ for opt in optMap.keys():
# Module option
if ":" in opt:
parts = opt.split(':')
- qvals = [ id, parts[0], parts[1], optMap[opt] ]
+ qvals = [id, parts[0], parts[1], optMap[opt]]
else:
- # Global option
- qvals = [ id, "GLOBAL", opt, optMap[opt] ]
+ # Global option
+ qvals = [id, "GLOBAL", opt, optMap[opt]]
try:
self.dbh.execute(qry, qvals)
- except psycopg2.Error as e:
- # Try updating if inserting failed
- self.conn.rollback()
- try:
- qry = "UPDATE tbl_scan_config \
- SET val = %s WHERE scan_instance_id = %s AND \
- component = %s AND opt = %s"
- nvals = [ qvals[3], qvals[0], qvals[1], qvals[2] ]
- self.dbh.execute(qry, nvals)
- except psycogp2.Error as e:
- self.sf.error("SQL error encountered when storing config, aborting: " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])
self.conn.commit()
# Retreive configuration data for a scan component
def scanConfigGet(self, instanceId):
qry = "SELECT component, opt, val FROM tbl_scan_config \
- WHERE scan_instance_id = %s ORDER BY component, opt"
+ WHERE scan_instance_id = ? ORDER BY component, opt"
qvars = [instanceId]
try:
retval = dict()
@@ -574,7 +551,7 @@ class SpiderFootDb:
else:
retval[component + ":" + opt] = val
return retval
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])
# Store an event
@@ -607,16 +584,16 @@ class SpiderFootDb:
if truncateSize > 0:
storeData = storeData[0:truncateSize]
- if sfEvent.sourceEventHash in [ "", None]:
+ if sfEvent.sourceEventHash in ["", None]:
self.sf.fatal("UNABLE TO CREATE RECORD WITH EMPTY SOURCE EVENT HASH!")
qry = "INSERT INTO tbl_scan_results \
(scan_instance_id, hash, type, generated, confidence, \
visibility, risk, module, data, source_event_hash) \
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
- qvals = [ instanceId, sfEvent.getHash(), sfEvent.eventType, sfEvent.generated,
- sfEvent.confidence, sfEvent.visibility, sfEvent.risk,
- sfEvent.module, storeData, sfEvent.sourceEventHash ]
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+ qvals = [instanceId, sfEvent.getHash(), sfEvent.eventType, sfEvent.generated,
+ sfEvent.confidence, sfEvent.visibility, sfEvent.risk,
+ sfEvent.module, storeData, sfEvent.sourceEventHash]
#print "STORING: " + str(qvals)
@@ -624,9 +601,8 @@ class SpiderFootDb:
self.dbh.execute(qry, qvals)
self.conn.commit()
return None
- except psycopg2.Error as e:
- self.sf.fatal("SQL error encountered when storing event data (" + str(self.dbh) + ": " +
- e.args[0])
+ except sqlite3.Error as e:
+ self.sf.fatal("SQL error encountered when storing event data (" + str(self.dbh) + ": " + e.args[0])
# List of all previously run scans
def scanInstanceList(self):
@@ -646,19 +622,19 @@ class SpiderFootDb:
try:
self.dbh.execute(qry)
return self.dbh.fetchall()
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.error("SQL error encountered when fetching scan list: " + e.args[0])
# History of data from the scan
def scanResultHistory(self, instanceId):
qry = "SELECT STRFTIME('%H:%M %w', generated, 'unixepoch') AS hourmin, \
type, COUNT(*) FROM tbl_scan_results \
- WHERE scan_instance_id = %s GROUP BY hourmin, type"
+ WHERE scan_instance_id = ? GROUP BY hourmin, type"
qvars = [instanceId]
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.error("SQL error encountered when fetching scan history: " + e.args[0])
@@ -667,23 +643,21 @@ class SpiderFootDb:
# the output of this needs to be aligned with scanResultEvent,
# as other functions call both expecting the same output.
qry = "SELECT ROUND(c.generated) AS generated, c.data, \
- s.data as source_data, \
+ s.data as 'source_data', \
c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \
c.source_event_hash, t.event_descr \
FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \
- WHERE c.scan_instance_id = %s AND c.source_event_hash = s.hash AND \
+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \
s.scan_instance_id = c.scan_instance_id AND \
t.event = c.type AND c.hash in ("
qvars = [instanceId]
for hashId in elementIdList:
qry = qry + "'" + hashId + "',"
- qry = qry + "'')"
+ qry += "'')"
try:
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
- except psycopg2.Error as e:
+ except sqlite3.Error as e:
self.sf.error("SQL error encountered when getting source element IDs: " + e.args[0])
-
-