Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sflib: Add annotations and docstrings (#1333)

bcoles0bd968941d4b
diff --git a/sflib.py b/sflib.py
index 598c5726..74605d25 100644
--- a/sflib.py
+++ b/sflib.py
@@ -53,13 +53,12 @@ class SpiderFoot:
socksProxy (str): SOCKS proxy
opts (dict): configuration options
"""
-
_dbh = None
_scanId = None
_socksProxy = None
opts = dict()
- def __init__(self, options):
+ def __init__(self, options: dict) -> None:
"""Initialize SpiderFoot object.
Args:
@@ -94,7 +93,7 @@ class SpiderFoot:
return self._dbh
@property
- def scanId(self):
+ def scanId(self) -> str:
"""Scan instance ID
Returns:
@@ -103,7 +102,7 @@ class SpiderFoot:
return self._scanId
@property
- def socksProxy(self):
+ def socksProxy(self) -> str:
"""SOCKS proxy
Returns:
@@ -123,16 +122,16 @@ class SpiderFoot:
self._dbh = dbh
@scanId.setter
- def scanId(self, scanId):
+ def scanId(self, scanId: str) -> str:
"""Set the scan ID this instance of SpiderFoot is being used in.
Args:
- scanId: scan instance ID
+ scanId (str): scan instance ID
"""
self._scanId = scanId
@socksProxy.setter
- def socksProxy(self, socksProxy):
+ def socksProxy(self, socksProxy: str) -> str:
"""SOCKS proxy
Bit of a hack to support SOCKS because of the loading order of
@@ -144,7 +143,7 @@ class SpiderFoot:
"""
self._socksProxy = socksProxy
- def optValueToData(self, val):
+ def optValueToData(self, val: str) -> str:
"""Supplied an option value, return the data based on what the
value is. If val is a URL, you'll get back the fetched content,
if val is a file path it will be loaded and get back the contents,
@@ -156,7 +155,6 @@ class SpiderFoot:
Returns:
str: option data
"""
-
if not isinstance(val, str):
self.error(f"Invalid option value {val}")
return None
@@ -185,50 +183,46 @@ class SpiderFoot:
return val
- def error(self, message):
+ def error(self, message: str) -> None:
"""Print and log an error message
Args:
message (str): error message
"""
-
if not self.opts['__logging']:
return
self.log.error(message, extra={'scanId': self._scanId})
- def fatal(self, error):
+ def fatal(self, error: str) -> None:
"""Print an error message and stacktrace then exit.
Args:
error (str): error message
"""
-
self.log.critical(error, extra={'scanId': self._scanId})
print(str(inspect.stack()))
sys.exit(-1)
- def status(self, message):
+ def status(self, message: str) -> None:
"""Log and print a status message.
Args:
message (str): status message
"""
-
if not self.opts['__logging']:
return
self.log.info(message, extra={'scanId': self._scanId})
- def info(self, message):
+ def info(self, message: str) -> None:
"""Log and print an info message.
Args:
message (str): info message
"""
-
if not self.opts['__logging']:
return
@@ -250,13 +244,12 @@ class SpiderFoot:
self.log.info(f"{modName} : {message}", extra={'scanId': self._scanId, 'component': modName})
- def debug(self, message):
+ def debug(self, message: str) -> None:
"""Log and print a debug message.
Args:
message (str): debug message
"""
-
if not self.opts['_debug']:
return
if not self.opts['__logging']:
@@ -280,9 +273,12 @@ class SpiderFoot:
self.log.debug(f"{modName} : {message}", extra={'scanId': self._scanId})
@staticmethod
- def myPath():
- # This will get us the program's directory, even if we are frozen using py2exe.
+ def myPath() -> str:
+ """This will get us the program's directory, even if we are frozen using py2exe.
+ Returns:
+ str: Program root directory
+ """
# Determine whether we've been compiled by py2exe
if hasattr(sys, "frozen"):
return os.path.dirname(sys.executable)
@@ -290,32 +286,30 @@ class SpiderFoot:
return os.path.dirname(__file__)
@classmethod
- def dataPath(cls):
+ def dataPath(cls) -> str:
"""Returns the file system location of SpiderFoot data and configuration files.
Returns:
str: SpiderFoot file system path
"""
-
path = os.environ.get('SPIDERFOOT_DATA')
return path if path is not None else cls.myPath()
- def hashstring(self, string):
+ def hashstring(self, string: str) -> str:
"""Returns a SHA256 hash of the specified input.
Args:
- string (str, list, dict): data to be hashed
+ string (str): data to be hashed
Returns:
str: SHA256 hash
"""
-
s = string
if type(string) in [list, dict]:
s = str(string)
return hashlib.sha256(s.encode('raw_unicode_escape')).hexdigest()
- def cachePath(self):
+ def cachePath(self) -> str:
"""Returns the file system location of the cacha data files.
Returns:
@@ -328,14 +322,13 @@ class SpiderFoot:
os.mkdir(path)
return path
- def cachePut(self, label, data):
+ def cachePut(self, label: str, data: str) -> None:
"""Store data to the cache
Args:
- label (str): TBD
- data (str): TBD
+ label (str): Name of the cached data to be used when retrieving the cached data.
+ data (str): Data to cache
"""
-
pathLabel = hashlib.sha224(label.encode('utf-8')).hexdigest()
cacheFile = self.cachePath() + "/" + pathLabel
with io.open(cacheFile, "w", encoding="utf-8", errors="ignore") as fp:
@@ -351,17 +344,17 @@ class SpiderFoot:
else:
fp.write(data)
- def cacheGet(self, label, timeoutHrs):
+ def cacheGet(self, label: str, timeoutHrs: int) -> str:
"""Retreive data from the cache
Args:
- label (str): TBD
- timeoutHrs (str): TBD
+ label (str): Name of the cached data to retrieve
+ timeoutHrs (int): Age of the cached data (in hours)
+ for which the data is considered to be too old and ignored.
Returns:
str: cached data
"""
-
if label is None:
return None
@@ -381,11 +374,11 @@ class SpiderFoot:
return None
- def configSerialize(self, opts, filterSystem=True):
+ def configSerialize(self, opts: dict, filterSystem: bool = True):
"""Convert a Python dictionary to something storable in the database.
Args:
- opts (dict): TBD
+ opts (dict): Dictionary of SpiderFoot configuration options
filterSystem (bool): TBD
Returns:
@@ -394,7 +387,6 @@ class SpiderFoot:
Raises:
TypeError: arg type was invalid
"""
-
if not isinstance(opts, dict):
raise TypeError("opts is %s; expected dict()" % type(opts))
@@ -446,16 +438,14 @@ class SpiderFoot:
return storeopts
- def configUnserialize(self, opts, referencePoint, filterSystem=True):
+ def configUnserialize(self, opts: dict, referencePoint: dict, filterSystem: bool = True):
"""Take strings, etc. from the database or UI and convert them
to a dictionary for Python to process.
- referencePoint is needed to know the actual types the options
- are supposed to be.
Args:
- opts (dict): TBD
- referencePoint (dict): TBD
- filterSystem (bool): TBD
+ opts (dict): SpiderFoot configuration options
+ referencePoint (dict): needed to know the actual types the options are supposed to be.
+ filterSystem (bool): Ignore global "system" configuration options
Returns:
dict: TBD
@@ -543,7 +533,7 @@ class SpiderFoot:
return returnOpts
- def modulesProducing(self, events):
+ def modulesProducing(self, events: list) -> list:
"""Return an array of modules that produce the list of types supplied.
Args:
@@ -577,7 +567,7 @@ class SpiderFoot:
return list(set(modlist))
- def modulesConsuming(self, events):
+ def modulesConsuming(self, events: list) -> list:
"""Return an array of modules that consume the list of types supplied.
Args:
@@ -612,7 +602,7 @@ class SpiderFoot:
return list(set(modlist))
- def eventsFromModules(self, modules):
+ def eventsFromModules(self, modules: list) -> list:
"""Return an array of types that are produced by the list of modules supplied.
Args:
@@ -640,7 +630,7 @@ class SpiderFoot:
return evtlist
- def eventsToModules(self, modules):
+ def eventsToModules(self, modules: list) -> list:
"""Return an array of types that are consumed by the list of modules supplied.
Args:
@@ -668,7 +658,7 @@ class SpiderFoot:
return evtlist
- def urlRelativeToAbsolute(self, url):
+ def urlRelativeToAbsolute(self, url: str) -> str:
"""Turn a relative path into an absolute path
Args:
@@ -677,7 +667,6 @@ class SpiderFoot:
Returns:
str: URL relative path
"""
-
if not url:
self.error("Invalid URL: %s" % url)
return None
@@ -706,7 +695,7 @@ class SpiderFoot:
return '/'.join(finalBits)
- def urlBaseDir(self, url):
+ def urlBaseDir(self, url: str) -> str:
"""Extract the top level directory from a URL
Args:
@@ -715,7 +704,6 @@ class SpiderFoot:
Returns:
str: base directory
"""
-
if not url:
self.error("Invalid URL: %s" % url)
return None
@@ -734,7 +722,7 @@ class SpiderFoot:
return base + '/'
- def urlBaseUrl(self, url):
+ def urlBaseUrl(self, url: str) -> str:
"""Extract the scheme and domain from a URL
Does not return the trailing slash! So you can do .endswith() checks.
@@ -745,7 +733,6 @@ class SpiderFoot:
Returns:
str: base URL without trailing slash
"""
-
if not url:
self.error("Invalid URL: %s" % url)
return None
@@ -760,7 +747,7 @@ class SpiderFoot:
return bits.group(1).lower()
- def urlFQDN(self, url):
+ def urlFQDN(self, url: str) -> str:
"""Extract the FQDN from a URL.
Args:
@@ -769,7 +756,6 @@ class SpiderFoot:
Returns:
str: FQDN
"""
-
if not url:
self.error(f"Invalid URL: {url}")
return None
@@ -783,17 +769,16 @@ class SpiderFoot:
# http://abc.com will split to ['http:', '', 'abc.com']
return baseurl.split('/')[count].lower()
- def domainKeyword(self, domain, tldList):
+ def domainKeyword(self, domain: str, tldList: list) -> str:
"""Extract the keyword (the domain without the TLD or any subdomains) from a domain.
Args:
domain (str): The domain to check.
- tldList (str): The list of TLDs based on the Mozilla public list.
+ tldList (list): The list of TLDs based on the Mozilla public list.
Returns:
str: The keyword
"""
-
if not domain:
self.error(f"Invalid domain: {domain}")
return None
@@ -812,17 +797,16 @@ class SpiderFoot:
return ret
- def domainKeywords(self, domainList, tldList):
+ def domainKeywords(self, domainList: list, tldList: list) -> set:
"""Extract the keywords (the domains without the TLD or any subdomains) from a list of domains.
Args:
domainList (list): The list of domains to check.
- tldList (str): The list of TLDs based on the Mozilla public list.
+ tldList (list): The list of TLDs based on the Mozilla public list.
Returns:
set: List of keywords
"""
-
if not domainList:
self.error("Invalid domain list: %s" % domainList)
return set()
@@ -834,17 +818,16 @@ class SpiderFoot:
self.debug("Keywords: %s" % keywords)
return set([k for k in keywords if k])
- def hostDomain(self, hostname, tldList):
+ def hostDomain(self, hostname: str, tldList: list) -> str:
"""Obtain the domain name for a supplied hostname.
Args:
hostname (str): The hostname to check.
- tldList (str): The list of TLDs based on the Mozilla public list.
+ tldList (list): The list of TLDs based on the Mozilla public list.
Returns:
str: The domain name.
"""
-
if not tldList:
return None
if not hostname:
@@ -853,7 +836,7 @@ class SpiderFoot:
ps = PublicSuffixList(tldList, only_icann=True)
return ps.privatesuffix(hostname)
- def validHost(self, hostname, tldList):
+ def validHost(self, hostname: str, tldList: str) -> bool:
"""Check if the provided string is a valid hostname with a valid public suffix TLD.
Args:
@@ -863,7 +846,6 @@ class SpiderFoot:
Returns:
bool
"""
-
if not tldList:
return False
if not hostname:
@@ -879,7 +861,7 @@ class SpiderFoot:
sfx = ps.privatesuffix(hostname)
return sfx is not None
- def isDomain(self, hostname, tldList):
+ def isDomain(self, hostname: str, tldList: list) -> bool:
"""Check if the provided hostname string is a valid domain name.
Given a possible hostname, check if it's a domain name
@@ -889,12 +871,11 @@ class SpiderFoot:
Args:
hostname (str): The hostname to check.
- tldList (str): The list of TLDs based on the Mozilla public list.
+ tldList (list): The list of TLDs based on the Mozilla public list.
Returns:
bool
"""
-
if not tldList:
return False
if not hostname:
@@ -904,7 +885,7 @@ class SpiderFoot:
sfx = ps.privatesuffix(hostname)
return sfx == hostname
- def validIP(self, address):
+ def validIP(self, address: str) -> bool:
"""Check if the provided string is a valid IPv4 address.
Args:
@@ -913,12 +894,11 @@ class SpiderFoot:
Returns:
bool
"""
-
if not address:
return False
return netaddr.valid_ipv4(address)
- def validIP6(self, address):
+ def validIP6(self, address: str) -> bool:
"""Check if the provided string is a valid IPv6 address.
Args:
@@ -927,12 +907,11 @@ class SpiderFoot:
Returns:
bool: string is a valid IPv6 address
"""
-
if not address:
return False
return netaddr.valid_ipv6(address)
- def validIpNetwork(self, cidr):
+ def validIpNetwork(self, cidr: str) -> bool:
"""Check if the provided string is a valid CIDR netblock.
Args:
@@ -952,7 +931,7 @@ class SpiderFoot:
except BaseException:
return False
- def isPublicIpAddress(self, ip):
+ def isPublicIpAddress(self, ip: str) -> bool:
"""Check if an IP address is public.
Args:
@@ -979,7 +958,7 @@ class SpiderFoot:
return False
return True
- def normalizeDNS(self, res):
+ def normalizeDNS(self, res: list) -> list:
"""Clean DNS results to be a simple list
Args:
@@ -988,7 +967,6 @@ class SpiderFoot:
Returns:
list: list of domains
"""
-
ret = list()
if not res:
@@ -1006,7 +984,7 @@ class SpiderFoot:
ret.append(host)
return ret
- def validEmail(self, email):
+ def validEmail(self, email: str) -> bool:
"""Check if the provided string is a valid email address.
Args:
@@ -1015,7 +993,6 @@ class SpiderFoot:
Returns:
bool: email is a valid email address
"""
-
if not isinstance(email, str):
return False
@@ -1038,7 +1015,7 @@ class SpiderFoot:
return True
- def validPhoneNumber(self, phone):
+ def validPhoneNumber(self, phone: str) -> bool:
"""Check if the provided string is a valid phone number.
Args:
@@ -1055,7 +1032,7 @@ class SpiderFoot:
except Exception:
return False
- def sanitiseInput(self, cmd):
+ def sanitiseInput(self, cmd: str) -> bool:
"""Verify input command is safe to execute
Args:
@@ -1064,7 +1041,6 @@ class SpiderFoot:
Returns:
bool: command is "safe"
"""
-
chars = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.']
@@ -1083,13 +1059,12 @@ class SpiderFoot:
return True
- def dictwords(self):
- """Return dictionary words and/or names from several language dictionaries
+ def dictwords(self) -> list:
+ """Return dictionary words and/or names from several language dictionaries.
Returns:
list: words and names from dictionaries
"""
-
wd = dict()
dicts = ["english", "german", "french", "spanish"]
@@ -1108,13 +1083,12 @@ class SpiderFoot:
return list(wd.keys())
- def dictnames(self):
+ def dictnames(self) -> list:
"""Return names of available dictionary files.
Returns:
list: list of dictionary file names.
"""
-
wd = dict()
dicts = ["names"]
@@ -1134,7 +1108,7 @@ class SpiderFoot:
return list(wd.keys())
- def resolveHost(self, host):
+ def resolveHost(self, host: str) -> list:
"""Return a normalised resolution of a hostname.
Args:
@@ -1143,7 +1117,6 @@ class SpiderFoot:
Returns:
list: IP addresses
"""
-
if not host:
self.error(f"Unable to resolve host: {host} (Invalid host)")
return list()
@@ -1162,7 +1135,7 @@ class SpiderFoot:
return list(set(addrs))
- def resolveIP(self, ipaddr):
+ def resolveIP(self, ipaddr: str) -> list:
"""Return a normalised resolution of an IPv4 address.
Args:
@@ -1192,7 +1165,7 @@ class SpiderFoot:
return list(set(addrs))
- def resolveHost6(self, hostname):
+ def resolveHost6(self, hostname: str) -> list:
"""Return a normalised resolution of an IPv6 address.
Args:
@@ -1201,7 +1174,6 @@ class SpiderFoot:
Returns:
list
"""
-
addrs = list()
if not hostname:
@@ -1221,7 +1193,7 @@ class SpiderFoot:
return list(set(addrs))
- def validateIP(self, host, ip):
+ def validateIP(self, host: str, ip: str) -> bool:
"""Verify a host resolves to a given IP.
Args:
@@ -1231,7 +1203,6 @@ class SpiderFoot:
Returns:
bool: host resolves to the given IP address
"""
-
addrs = self.resolveHost(host)
if not addrs:
@@ -1243,7 +1214,7 @@ class SpiderFoot:
return False
- def resolveTargets(self, target, validateReverse):
+ def resolveTargets(self, target, validateReverse: bool) -> list:
"""Resolve alternative names for a given target.
Args:
@@ -1291,12 +1262,12 @@ class SpiderFoot:
ret.extend(names)
return list(set(ret))
- def safeSocket(self, host, port, timeout):
+ def safeSocket(self, host: str, port: int, timeout: int) -> 'ssl.SSLSocket':
"""Create a safe socket that's using SOCKS/TOR if it was enabled.
Args:
host (str): host
- port (str): port
+ port (int): port
timeout (int): timeout
Returns:
@@ -1306,12 +1277,12 @@ class SpiderFoot:
sock.settimeout(int(timeout))
return sock
- def safeSSLSocket(self, host, port, timeout):
+ def safeSSLSocket(self, host: str, port: int, timeout: int) -> 'ssl.SSLSocket':
"""Create a safe SSL connection that's using SOCKs/TOR if it was enabled.
Args:
host (str): host
- port (str): port
+ port (int): port
timeout (int): timeout
Returns:
@@ -1324,7 +1295,7 @@ class SpiderFoot:
sock.do_handshake()
return sock
- def parseRobotsTxt(self, robotsTxtData):
+ def parseRobotsTxt(self, robotsTxtData: str) -> list:
"""Parse the contents of robots.txt.
Args:
@@ -1338,7 +1309,6 @@ class SpiderFoot:
fix whitespace parsing; ie, " " is not a valid disallowed path
"""
-
returnArr = list()
if not isinstance(robotsTxtData, str):
@@ -1353,7 +1323,7 @@ class SpiderFoot:
return returnArr
- def parseHashes(self, data):
+ def parseHashes(self, data: str) -> list:
"""Extract all hashes within the supplied content.
Args:
@@ -1362,7 +1332,6 @@ class SpiderFoot:
Returns:
list: list of hashes
"""
-
ret = list()
if not isinstance(data, str):
@@ -1383,7 +1352,7 @@ class SpiderFoot:
return ret
- def parseEmails(self, data):
+ def parseEmails(self, data: str) -> list:
"""Extract all email addresses within the supplied content.
Args:
@@ -1392,7 +1361,6 @@ class SpiderFoot:
Returns:
list: list of email addresses
"""
-
if not isinstance(data, str):
return list()
@@ -1405,7 +1373,7 @@ class SpiderFoot:
return list(emails)
- def parseCreditCards(self, data):
+ def parseCreditCards(self, data: str) -> list:
"""Find all credit card numbers with the supplied content.
Extracts numbers with lengths ranging from 13 - 19 digits
@@ -1419,7 +1387,6 @@ class SpiderFoot:
Returns:
list: list of credit card numbers
"""
-
if not isinstance(data, str):
return list()
@@ -1435,7 +1402,6 @@ class SpiderFoot:
# Verify each extracted number using Luhn's algorithm
for match in matches:
-
if int(match) == 0:
continue
@@ -1457,7 +1423,7 @@ class SpiderFoot:
creditCards.add(match)
return list(creditCards)
- def getCountryCodeDict(self):
+ def getCountryCodeDict(self) -> dict:
"""Dictionary of country codes and associated country names.
Returns:
@@ -1724,7 +1690,7 @@ class SpiderFoot:
"UK": "United Kingdom"
}
- def countryNameFromCountryCode(self, countryCode):
+ def countryNameFromCountryCode(self, countryCode: str) -> str:
"""Convert a country code to full country name
Args:
@@ -1738,7 +1704,7 @@ class SpiderFoot:
return self.getCountryCodeDict().get(countryCode.upper())
- def countryNameFromTld(self, tld):
+ def countryNameFromTld(self, tld: str) -> str:
"""Retrieve the country name associated with a TLD.
Args:
@@ -1747,7 +1713,6 @@ class SpiderFoot:
Returns:
str: country name
"""
-
if not isinstance(tld, str):
return None
@@ -1772,7 +1737,7 @@ class SpiderFoot:
return None
- def parseIBANNumbers(self, data):
+ def parseIBANNumbers(self, data: str) -> list:
"""Find all International Bank Account Numbers (IBANs) within the supplied content.
Extracts possible IBANs using a generic regex.
@@ -1854,7 +1819,7 @@ class SpiderFoot:
return list(ibans)
- def sslDerToPem(self, der_cert):
+ def sslDerToPem(self, der_cert: bytes) -> str:
"""Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded string version of the same certificate.
Args:
@@ -1872,18 +1837,17 @@ class SpiderFoot:
return ssl.DER_cert_to_PEM_cert(der_cert)
- def parseCert(self, rawcert, fqdn=None, expiringdays=30):
+ def parseCert(self, rawcert: str, fqdn: str = None, expiringdays: int = 30) -> dict:
"""Parse a PEM-format SSL certificate.
Args:
- rawcert (str): TBD
+ rawcert (str): PEM-format SSL certificate
fqdn (str): TBD
- expiringdays (int): TBD
+ expiringdays (int): The certificate will be considered as "expiring" if within this number of days of expiry.
Returns:
dict: certificate details
"""
-
if not rawcert:
self.error(f"Invalid certificate: {rawcert}")
return None
@@ -1982,7 +1946,7 @@ class SpiderFoot:
return ret
- def extractUrls(self, content):
+ def extractUrls(self, content: str) -> list:
"""Extract all URLs from a string.
Args:
@@ -1995,7 +1959,7 @@ class SpiderFoot:
# https://tools.ietf.org/html/rfc3986#section-3.3
return re.findall(r"(https?://[a-zA-Z0-9-\.:]+/[\-\._~!\$&'\(\)\*\+\,\;=:@/a-zA-Z0-9]*)", html.unescape(content))
- def parseLinks(self, url, data, domains):
+ def parseLinks(self, url: str, data: str, domains: list) -> list:
"""Find all URLs within the supplied content.
This does not fetch any URLs!
@@ -2007,7 +1971,7 @@ class SpiderFoot:
be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'
Args:
- url (str): TBD
+ url (str): base URL used to construct absolute URLs from relative URLs
data (str): data to examine for links
domains: TBD
@@ -2114,10 +2078,23 @@ class SpiderFoot:
return returnLinks
- def urlEncodeUnicode(self, url):
+ def urlEncodeUnicode(self, url: str) -> str:
+ """Encode a string as unicode.
+
+ Args:
+ url (str): URL to encode
+
+ Returns:
+ str: unicode string
+ """
return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), url)
- def getSession(self):
+ def getSession(self) -> 'requests.sessions.Session':
+ """Return requests session object.
+
+ Returns:
+ requests.sessions.Session: requests session
+ """
session = requests.session()
if self.socksProxy:
session.proxies = {
@@ -2126,16 +2103,17 @@ class SpiderFoot:
}
return session
- def removeUrlCreds(self, url):
- """Remove key= and others from URLs to avoid credentials in logs.
+ def removeUrlCreds(self, url: str) -> str:
+ """Remove potentially sensitive strings (such as "key=..." and "password=...") from a string.
+
+ Used to remove potential credentials from URLs prior during logging.
Args:
url (str): URL
Returns:
- str: URL
+ str: Sanitized URL
"""
-
pats = {
r'key=\S+': "key=XXX",
r'pass=\S+': "pass=XXX",
@@ -2149,7 +2127,7 @@ class SpiderFoot:
return ret
- def useProxyForUrl(self, url):
+ def useProxyForUrl(self, url: str) -> bool:
"""Check if the configured proxy should be used to connect to a specified URL.
Args:
@@ -2201,20 +2179,20 @@ class SpiderFoot:
def fetchUrl(
self,
- url,
- fatal=False,
- cookies=None,
- timeout=30,
- useragent="SpiderFoot",
- headers=None,
- noLog=False,
- postData=None,
- dontMangle=False,
- sizeLimit=None,
- headOnly=False,
- verify=True
- ):
- """Fetch a URL, return the response object.
+ url: str,
+ fatal: bool = False,
+ cookies: str = None,
+ timeout: int = 30,
+ useragent: str = "SpiderFoot",
+ headers: dict = None,
+ noLog: bool = False,
+ postData: str = None,
+ dontMangle: bool = False,
+ sizeLimit: int = None,
+ headOnly: bool = False,
+ verify: bool = True
+ ) -> dict:
+ """Fetch a URL and return the HTTP response as a dictionary.
Args:
url (str): URL to fetch
@@ -2222,7 +2200,7 @@ class SpiderFoot:
cookies (str): cookies
timeout (int): timeout
useragent (str): user agent header
- headers (str): headers
+ headers (dict): headers
noLog (bool): do not log request
postData (str): HTTP POST data
dontMangle (bool): do not mangle
@@ -2233,7 +2211,6 @@ class SpiderFoot:
Returns:
dict: HTTP response
"""
-
if not url:
return None
@@ -2462,16 +2439,15 @@ class SpiderFoot:
self.info(f"Fetched {self.removeUrlCreds(url)} ({len(result['content'] or '')} bytes in {t}s)")
return result
- def checkDnsWildcard(self, target):
- """Check if wildcard DNS is enabled by looking up a random hostname
+ def checkDnsWildcard(self, target: str) -> bool:
+ """Check if wildcard DNS is enabled by looking up a random subdomain.
Args:
target (str): TBD
Returns:
- bool: wildcard DNS
+ bool: Domain returns DNS records for any subdomains
"""
-
if not target:
return False
@@ -2483,7 +2459,7 @@ class SpiderFoot:
return True
- def googleIterate(self, searchString, opts={}):
+ def googleIterate(self, searchString: str, opts: dict = {}) -> dict:
"""Request search results from the Google API.
Will return a dict:
@@ -2501,7 +2477,7 @@ class SpiderFoot:
opts (dict): TBD
Returns:
- dict: TBD
+ dict: Search results as {"webSearchUrl": "URL", "urls": [results]}
"""
search_string = searchString.replace(" ", "%20")
@@ -2542,7 +2518,7 @@ class SpiderFoot:
"webSearchUrl": f"https://www.google.com/search?q={search_string}&{params}"
}
- def bingIterate(self, searchString, opts={}):
+ def bingIterate(self, searchString: str, opts: dict = {}) -> dict:
"""Request search results from the Bing API.
Will return a dict:
@@ -2561,7 +2537,7 @@ class SpiderFoot:
opts (dict): TBD
Returns:
- dict: TBD
+ dict: Search results as {"webSearchUrl": "URL", "urls": [results]}
"""
search_string = searchString.replace(" ", "%20")