SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sflib: Add annotations and docstrings (#1333)
bcoles0bd968941d4b
diff --git a/sflib.py b/sflib.pyindex 598c5726..74605d25 100644--- a/sflib.py+++ b/sflib.py@@ -53,13 +53,12 @@ class SpiderFoot:socksProxy (str): SOCKS proxyopts (dict): configuration options"""-_dbh = None_scanId = None_socksProxy = Noneopts = dict()- def __init__(self, options):+ def __init__(self, options: dict) -> None:"""Initialize SpiderFoot object.Args:@@ -94,7 +93,7 @@ class SpiderFoot:return self._dbh@property- def scanId(self):+ def scanId(self) -> str:"""Scan instance IDReturns:@@ -103,7 +102,7 @@ class SpiderFoot:return self._scanId@property- def socksProxy(self):+ def socksProxy(self) -> str:"""SOCKS proxyReturns:@@ -123,16 +122,16 @@ class SpiderFoot:self._dbh = dbh@scanId.setter- def scanId(self, scanId):+ def scanId(self, scanId: str) -> str:"""Set the scan ID this instance of SpiderFoot is being used in.Args:- scanId: scan instance ID+ scanId (str): scan instance ID"""self._scanId = scanId@socksProxy.setter- def socksProxy(self, socksProxy):+ def socksProxy(self, socksProxy: str) -> str:"""SOCKS proxyBit of a hack to support SOCKS because of the loading order of@@ -144,7 +143,7 @@ class SpiderFoot:"""self._socksProxy = socksProxy- def optValueToData(self, val):+ def optValueToData(self, val: str) -> str:"""Supplied an option value, return the data based on what thevalue is. If val is a URL, you'll get back the fetched content,if val is a file path it will be loaded and get back the contents,@@ -156,7 +155,6 @@ class SpiderFoot:Returns:str: option data"""-if not isinstance(val, str):self.error(f"Invalid option value {val}")return None@@ -185,50 +183,46 @@ class SpiderFoot:return val- def error(self, message):+ def error(self, message: str) -> None:"""Print and log an error messageArgs:message (str): error message"""-if not self.opts['__logging']:returnself.log.error(message, extra={'scanId': self._scanId})- def fatal(self, error):+ def fatal(self, error: str) -> None:"""Print an error message and stacktrace then exit.Args:error (str): error message"""-self.log.critical(error, extra={'scanId': self._scanId})print(str(inspect.stack()))sys.exit(-1)- def status(self, message):+ def status(self, message: str) -> None:"""Log and print a status message.Args:message (str): status message"""-if not self.opts['__logging']:returnself.log.info(message, extra={'scanId': self._scanId})- def info(self, message):+ def info(self, message: str) -> None:"""Log and print an info message.Args:message (str): info message"""-if not self.opts['__logging']:return@@ -250,13 +244,12 @@ class SpiderFoot:self.log.info(f"{modName} : {message}", extra={'scanId': self._scanId, 'component': modName})- def debug(self, message):+ def debug(self, message: str) -> None:"""Log and print a debug message.Args:message (str): debug message"""-if not self.opts['_debug']:returnif not self.opts['__logging']:@@ -280,9 +273,12 @@ class SpiderFoot:self.log.debug(f"{modName} : {message}", extra={'scanId': self._scanId})@staticmethod- def myPath():- # This will get us the program's directory, even if we are frozen using py2exe.+ def myPath() -> str:+ """This will get us the program's directory, even if we are frozen using py2exe.+ Returns:+ str: Program root directory+ """# Determine whether we've been compiled by py2exeif hasattr(sys, "frozen"):return os.path.dirname(sys.executable)@@ -290,32 +286,30 @@ class SpiderFoot:return os.path.dirname(__file__)@classmethod- def dataPath(cls):+ def dataPath(cls) -> str:"""Returns the file system location of SpiderFoot data and configuration files.Returns:str: SpiderFoot file system path"""-path = os.environ.get('SPIDERFOOT_DATA')return path if path is not None else cls.myPath()- def hashstring(self, string):+ def hashstring(self, string: str) -> str:"""Returns a SHA256 hash of the specified input.Args:- string (str, list, dict): data to be hashed+ string (str): data to be hashedReturns:str: SHA256 hash"""-s = stringif type(string) in [list, dict]:s = str(string)return hashlib.sha256(s.encode('raw_unicode_escape')).hexdigest()- def cachePath(self):+ def cachePath(self) -> str:"""Returns the file system location of the cacha data files.Returns:@@ -328,14 +322,13 @@ class SpiderFoot:os.mkdir(path)return path- def cachePut(self, label, data):+ def cachePut(self, label: str, data: str) -> None:"""Store data to the cacheArgs:- label (str): TBD- data (str): TBD+ label (str): Name of the cached data to be used when retrieving the cached data.+ data (str): Data to cache"""-pathLabel = hashlib.sha224(label.encode('utf-8')).hexdigest()cacheFile = self.cachePath() + "/" + pathLabelwith io.open(cacheFile, "w", encoding="utf-8", errors="ignore") as fp:@@ -351,17 +344,17 @@ class SpiderFoot:else:fp.write(data)- def cacheGet(self, label, timeoutHrs):+ def cacheGet(self, label: str, timeoutHrs: int) -> str:"""Retreive data from the cacheArgs:- label (str): TBD- timeoutHrs (str): TBD+ label (str): Name of the cached data to retrieve+ timeoutHrs (int): Age of the cached data (in hours)+ for which the data is considered to be too old and ignored.Returns:str: cached data"""-if label is None:return None@@ -381,11 +374,11 @@ class SpiderFoot:return None- def configSerialize(self, opts, filterSystem=True):+ def configSerialize(self, opts: dict, filterSystem: bool = True):"""Convert a Python dictionary to something storable in the database.Args:- opts (dict): TBD+ opts (dict): Dictionary of SpiderFoot configuration optionsfilterSystem (bool): TBDReturns:@@ -394,7 +387,6 @@ class SpiderFoot:Raises:TypeError: arg type was invalid"""-if not isinstance(opts, dict):raise TypeError("opts is %s; expected dict()" % type(opts))@@ -446,16 +438,14 @@ class SpiderFoot:return storeopts- def configUnserialize(self, opts, referencePoint, filterSystem=True):+ def configUnserialize(self, opts: dict, referencePoint: dict, filterSystem: bool = True):"""Take strings, etc. from the database or UI and convert themto a dictionary for Python to process.- referencePoint is needed to know the actual types the options- are supposed to be.Args:- opts (dict): TBD- referencePoint (dict): TBD- filterSystem (bool): TBD+ opts (dict): SpiderFoot configuration options+ referencePoint (dict): needed to know the actual types the options are supposed to be.+ filterSystem (bool): Ignore global "system" configuration optionsReturns:dict: TBD@@ -543,7 +533,7 @@ class SpiderFoot:return returnOpts- def modulesProducing(self, events):+ def modulesProducing(self, events: list) -> list:"""Return an array of modules that produce the list of types supplied.Args:@@ -577,7 +567,7 @@ class SpiderFoot:return list(set(modlist))- def modulesConsuming(self, events):+ def modulesConsuming(self, events: list) -> list:"""Return an array of modules that consume the list of types supplied.Args:@@ -612,7 +602,7 @@ class SpiderFoot:return list(set(modlist))- def eventsFromModules(self, modules):+ def eventsFromModules(self, modules: list) -> list:"""Return an array of types that are produced by the list of modules supplied.Args:@@ -640,7 +630,7 @@ class SpiderFoot:return evtlist- def eventsToModules(self, modules):+ def eventsToModules(self, modules: list) -> list:"""Return an array of types that are consumed by the list of modules supplied.Args:@@ -668,7 +658,7 @@ class SpiderFoot:return evtlist- def urlRelativeToAbsolute(self, url):+ def urlRelativeToAbsolute(self, url: str) -> str:"""Turn a relative path into an absolute pathArgs:@@ -677,7 +667,6 @@ class SpiderFoot:Returns:str: URL relative path"""-if not url:self.error("Invalid URL: %s" % url)return None@@ -706,7 +695,7 @@ class SpiderFoot:return '/'.join(finalBits)- def urlBaseDir(self, url):+ def urlBaseDir(self, url: str) -> str:"""Extract the top level directory from a URLArgs:@@ -715,7 +704,6 @@ class SpiderFoot:Returns:str: base directory"""-if not url:self.error("Invalid URL: %s" % url)return None@@ -734,7 +722,7 @@ class SpiderFoot:return base + '/'- def urlBaseUrl(self, url):+ def urlBaseUrl(self, url: str) -> str:"""Extract the scheme and domain from a URLDoes not return the trailing slash! So you can do .endswith() checks.@@ -745,7 +733,6 @@ class SpiderFoot:Returns:str: base URL without trailing slash"""-if not url:self.error("Invalid URL: %s" % url)return None@@ -760,7 +747,7 @@ class SpiderFoot:return bits.group(1).lower()- def urlFQDN(self, url):+ def urlFQDN(self, url: str) -> str:"""Extract the FQDN from a URL.Args:@@ -769,7 +756,6 @@ class SpiderFoot:Returns:str: FQDN"""-if not url:self.error(f"Invalid URL: {url}")return None@@ -783,17 +769,16 @@ class SpiderFoot:# http://abc.com will split to ['http:', '', 'abc.com']return baseurl.split('/')[count].lower()- def domainKeyword(self, domain, tldList):+ def domainKeyword(self, domain: str, tldList: list) -> str:"""Extract the keyword (the domain without the TLD or any subdomains) from a domain.Args:domain (str): The domain to check.- tldList (str): The list of TLDs based on the Mozilla public list.+ tldList (list): The list of TLDs based on the Mozilla public list.Returns:str: The keyword"""-if not domain:self.error(f"Invalid domain: {domain}")return None@@ -812,17 +797,16 @@ class SpiderFoot:return ret- def domainKeywords(self, domainList, tldList):+ def domainKeywords(self, domainList: list, tldList: list) -> set:"""Extract the keywords (the domains without the TLD or any subdomains) from a list of domains.Args:domainList (list): The list of domains to check.- tldList (str): The list of TLDs based on the Mozilla public list.+ tldList (list): The list of TLDs based on the Mozilla public list.Returns:set: List of keywords"""-if not domainList:self.error("Invalid domain list: %s" % domainList)return set()@@ -834,17 +818,16 @@ class SpiderFoot:self.debug("Keywords: %s" % keywords)return set([k for k in keywords if k])- def hostDomain(self, hostname, tldList):+ def hostDomain(self, hostname: str, tldList: list) -> str:"""Obtain the domain name for a supplied hostname.Args:hostname (str): The hostname to check.- tldList (str): The list of TLDs based on the Mozilla public list.+ tldList (list): The list of TLDs based on the Mozilla public list.Returns:str: The domain name."""-if not tldList:return Noneif not hostname:@@ -853,7 +836,7 @@ class SpiderFoot:ps = PublicSuffixList(tldList, only_icann=True)return ps.privatesuffix(hostname)- def validHost(self, hostname, tldList):+ def validHost(self, hostname: str, tldList: str) -> bool:"""Check if the provided string is a valid hostname with a valid public suffix TLD.Args:@@ -863,7 +846,6 @@ class SpiderFoot:Returns:bool"""-if not tldList:return Falseif not hostname:@@ -879,7 +861,7 @@ class SpiderFoot:sfx = ps.privatesuffix(hostname)return sfx is not None- def isDomain(self, hostname, tldList):+ def isDomain(self, hostname: str, tldList: list) -> bool:"""Check if the provided hostname string is a valid domain name.Given a possible hostname, check if it's a domain name@@ -889,12 +871,11 @@ class SpiderFoot:Args:hostname (str): The hostname to check.- tldList (str): The list of TLDs based on the Mozilla public list.+ tldList (list): The list of TLDs based on the Mozilla public list.Returns:bool"""-if not tldList:return Falseif not hostname:@@ -904,7 +885,7 @@ class SpiderFoot:sfx = ps.privatesuffix(hostname)return sfx == hostname- def validIP(self, address):+ def validIP(self, address: str) -> bool:"""Check if the provided string is a valid IPv4 address.Args:@@ -913,12 +894,11 @@ class SpiderFoot:Returns:bool"""-if not address:return Falsereturn netaddr.valid_ipv4(address)- def validIP6(self, address):+ def validIP6(self, address: str) -> bool:"""Check if the provided string is a valid IPv6 address.Args:@@ -927,12 +907,11 @@ class SpiderFoot:Returns:bool: string is a valid IPv6 address"""-if not address:return Falsereturn netaddr.valid_ipv6(address)- def validIpNetwork(self, cidr):+ def validIpNetwork(self, cidr: str) -> bool:"""Check if the provided string is a valid CIDR netblock.Args:@@ -952,7 +931,7 @@ class SpiderFoot:except BaseException:return False- def isPublicIpAddress(self, ip):+ def isPublicIpAddress(self, ip: str) -> bool:"""Check if an IP address is public.Args:@@ -979,7 +958,7 @@ class SpiderFoot:return Falsereturn True- def normalizeDNS(self, res):+ def normalizeDNS(self, res: list) -> list:"""Clean DNS results to be a simple listArgs:@@ -988,7 +967,6 @@ class SpiderFoot:Returns:list: list of domains"""-ret = list()if not res:@@ -1006,7 +984,7 @@ class SpiderFoot:ret.append(host)return ret- def validEmail(self, email):+ def validEmail(self, email: str) -> bool:"""Check if the provided string is a valid email address.Args:@@ -1015,7 +993,6 @@ class SpiderFoot:Returns:bool: email is a valid email address"""-if not isinstance(email, str):return False@@ -1038,7 +1015,7 @@ class SpiderFoot:return True- def validPhoneNumber(self, phone):+ def validPhoneNumber(self, phone: str) -> bool:"""Check if the provided string is a valid phone number.Args:@@ -1055,7 +1032,7 @@ class SpiderFoot:except Exception:return False- def sanitiseInput(self, cmd):+ def sanitiseInput(self, cmd: str) -> bool:"""Verify input command is safe to executeArgs:@@ -1064,7 +1041,6 @@ class SpiderFoot:Returns:bool: command is "safe""""-chars = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm','n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z','0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.']@@ -1083,13 +1059,12 @@ class SpiderFoot:return True- def dictwords(self):- """Return dictionary words and/or names from several language dictionaries+ def dictwords(self) -> list:+ """Return dictionary words and/or names from several language dictionaries.Returns:list: words and names from dictionaries"""-wd = dict()dicts = ["english", "german", "french", "spanish"]@@ -1108,13 +1083,12 @@ class SpiderFoot:return list(wd.keys())- def dictnames(self):+ def dictnames(self) -> list:"""Return names of available dictionary files.Returns:list: list of dictionary file names."""-wd = dict()dicts = ["names"]@@ -1134,7 +1108,7 @@ class SpiderFoot:return list(wd.keys())- def resolveHost(self, host):+ def resolveHost(self, host: str) -> list:"""Return a normalised resolution of a hostname.Args:@@ -1143,7 +1117,6 @@ class SpiderFoot:Returns:list: IP addresses"""-if not host:self.error(f"Unable to resolve host: {host} (Invalid host)")return list()@@ -1162,7 +1135,7 @@ class SpiderFoot:return list(set(addrs))- def resolveIP(self, ipaddr):+ def resolveIP(self, ipaddr: str) -> list:"""Return a normalised resolution of an IPv4 address.Args:@@ -1192,7 +1165,7 @@ class SpiderFoot:return list(set(addrs))- def resolveHost6(self, hostname):+ def resolveHost6(self, hostname: str) -> list:"""Return a normalised resolution of an IPv6 address.Args:@@ -1201,7 +1174,6 @@ class SpiderFoot:Returns:list"""-addrs = list()if not hostname:@@ -1221,7 +1193,7 @@ class SpiderFoot:return list(set(addrs))- def validateIP(self, host, ip):+ def validateIP(self, host: str, ip: str) -> bool:"""Verify a host resolves to a given IP.Args:@@ -1231,7 +1203,6 @@ class SpiderFoot:Returns:bool: host resolves to the given IP address"""-addrs = self.resolveHost(host)if not addrs:@@ -1243,7 +1214,7 @@ class SpiderFoot:return False- def resolveTargets(self, target, validateReverse):+ def resolveTargets(self, target, validateReverse: bool) -> list:"""Resolve alternative names for a given target.Args:@@ -1291,12 +1262,12 @@ class SpiderFoot:ret.extend(names)return list(set(ret))- def safeSocket(self, host, port, timeout):+ def safeSocket(self, host: str, port: int, timeout: int) -> 'ssl.SSLSocket':"""Create a safe socket that's using SOCKS/TOR if it was enabled.Args:host (str): host- port (str): port+ port (int): porttimeout (int): timeoutReturns:@@ -1306,12 +1277,12 @@ class SpiderFoot:sock.settimeout(int(timeout))return sock- def safeSSLSocket(self, host, port, timeout):+ def safeSSLSocket(self, host: str, port: int, timeout: int) -> 'ssl.SSLSocket':"""Create a safe SSL connection that's using SOCKs/TOR if it was enabled.Args:host (str): host- port (str): port+ port (int): porttimeout (int): timeoutReturns:@@ -1324,7 +1295,7 @@ class SpiderFoot:sock.do_handshake()return sock- def parseRobotsTxt(self, robotsTxtData):+ def parseRobotsTxt(self, robotsTxtData: str) -> list:"""Parse the contents of robots.txt.Args:@@ -1338,7 +1309,6 @@ class SpiderFoot:fix whitespace parsing; ie, " " is not a valid disallowed path"""-returnArr = list()if not isinstance(robotsTxtData, str):@@ -1353,7 +1323,7 @@ class SpiderFoot:return returnArr- def parseHashes(self, data):+ def parseHashes(self, data: str) -> list:"""Extract all hashes within the supplied content.Args:@@ -1362,7 +1332,6 @@ class SpiderFoot:Returns:list: list of hashes"""-ret = list()if not isinstance(data, str):@@ -1383,7 +1352,7 @@ class SpiderFoot:return ret- def parseEmails(self, data):+ def parseEmails(self, data: str) -> list:"""Extract all email addresses within the supplied content.Args:@@ -1392,7 +1361,6 @@ class SpiderFoot:Returns:list: list of email addresses"""-if not isinstance(data, str):return list()@@ -1405,7 +1373,7 @@ class SpiderFoot:return list(emails)- def parseCreditCards(self, data):+ def parseCreditCards(self, data: str) -> list:"""Find all credit card numbers with the supplied content.Extracts numbers with lengths ranging from 13 - 19 digits@@ -1419,7 +1387,6 @@ class SpiderFoot:Returns:list: list of credit card numbers"""-if not isinstance(data, str):return list()@@ -1435,7 +1402,6 @@ class SpiderFoot:# Verify each extracted number using Luhn's algorithmfor match in matches:-if int(match) == 0:continue@@ -1457,7 +1423,7 @@ class SpiderFoot:creditCards.add(match)return list(creditCards)- def getCountryCodeDict(self):+ def getCountryCodeDict(self) -> dict:"""Dictionary of country codes and associated country names.Returns:@@ -1724,7 +1690,7 @@ class SpiderFoot:"UK": "United Kingdom"}- def countryNameFromCountryCode(self, countryCode):+ def countryNameFromCountryCode(self, countryCode: str) -> str:"""Convert a country code to full country nameArgs:@@ -1738,7 +1704,7 @@ class SpiderFoot:return self.getCountryCodeDict().get(countryCode.upper())- def countryNameFromTld(self, tld):+ def countryNameFromTld(self, tld: str) -> str:"""Retrieve the country name associated with a TLD.Args:@@ -1747,7 +1713,6 @@ class SpiderFoot:Returns:str: country name"""-if not isinstance(tld, str):return None@@ -1772,7 +1737,7 @@ class SpiderFoot:return None- def parseIBANNumbers(self, data):+ def parseIBANNumbers(self, data: str) -> list:"""Find all International Bank Account Numbers (IBANs) within the supplied content.Extracts possible IBANs using a generic regex.@@ -1854,7 +1819,7 @@ class SpiderFoot:return list(ibans)- def sslDerToPem(self, der_cert):+ def sslDerToPem(self, der_cert: bytes) -> str:"""Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded string version of the same certificate.Args:@@ -1872,18 +1837,17 @@ class SpiderFoot:return ssl.DER_cert_to_PEM_cert(der_cert)- def parseCert(self, rawcert, fqdn=None, expiringdays=30):+ def parseCert(self, rawcert: str, fqdn: str = None, expiringdays: int = 30) -> dict:"""Parse a PEM-format SSL certificate.Args:- rawcert (str): TBD+ rawcert (str): PEM-format SSL certificatefqdn (str): TBD- expiringdays (int): TBD+ expiringdays (int): The certificate will be considered as "expiring" if within this number of days of expiry.Returns:dict: certificate details"""-if not rawcert:self.error(f"Invalid certificate: {rawcert}")return None@@ -1982,7 +1946,7 @@ class SpiderFoot:return ret- def extractUrls(self, content):+ def extractUrls(self, content: str) -> list:"""Extract all URLs from a string.Args:@@ -1995,7 +1959,7 @@ class SpiderFoot:# https://tools.ietf.org/html/rfc3986#section-3.3return re.findall(r"(https?://[a-zA-Z0-9-\.:]+/[\-\._~!\$&'\(\)\*\+\,\;=:@/a-zA-Z0-9]*)", html.unescape(content))- def parseLinks(self, url, data, domains):+ def parseLinks(self, url: str, data: str, domains: list) -> list:"""Find all URLs within the supplied content.This does not fetch any URLs!@@ -2007,7 +1971,7 @@ class SpiderFoot:be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'Args:- url (str): TBD+ url (str): base URL used to construct absolute URLs from relative URLsdata (str): data to examine for linksdomains: TBD@@ -2114,10 +2078,23 @@ class SpiderFoot:return returnLinks- def urlEncodeUnicode(self, url):+ def urlEncodeUnicode(self, url: str) -> str:+ """Encode a string as unicode.++ Args:+ url (str): URL to encode++ Returns:+ str: unicode string+ """return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), url)- def getSession(self):+ def getSession(self) -> 'requests.sessions.Session':+ """Return requests session object.++ Returns:+ requests.sessions.Session: requests session+ """session = requests.session()if self.socksProxy:session.proxies = {@@ -2126,16 +2103,17 @@ class SpiderFoot:}return session- def removeUrlCreds(self, url):- """Remove key= and others from URLs to avoid credentials in logs.+ def removeUrlCreds(self, url: str) -> str:+ """Remove potentially sensitive strings (such as "key=..." and "password=...") from a string.++ Used to remove potential credentials from URLs prior during logging.Args:url (str): URLReturns:- str: URL+ str: Sanitized URL"""-pats = {r'key=\S+': "key=XXX",r'pass=\S+': "pass=XXX",@@ -2149,7 +2127,7 @@ class SpiderFoot:return ret- def useProxyForUrl(self, url):+ def useProxyForUrl(self, url: str) -> bool:"""Check if the configured proxy should be used to connect to a specified URL.Args:@@ -2201,20 +2179,20 @@ class SpiderFoot:def fetchUrl(self,- url,- fatal=False,- cookies=None,- timeout=30,- useragent="SpiderFoot",- headers=None,- noLog=False,- postData=None,- dontMangle=False,- sizeLimit=None,- headOnly=False,- verify=True- ):- """Fetch a URL, return the response object.+ url: str,+ fatal: bool = False,+ cookies: str = None,+ timeout: int = 30,+ useragent: str = "SpiderFoot",+ headers: dict = None,+ noLog: bool = False,+ postData: str = None,+ dontMangle: bool = False,+ sizeLimit: int = None,+ headOnly: bool = False,+ verify: bool = True+ ) -> dict:+ """Fetch a URL and return the HTTP response as a dictionary.Args:url (str): URL to fetch@@ -2222,7 +2200,7 @@ class SpiderFoot:cookies (str): cookiestimeout (int): timeoutuseragent (str): user agent header- headers (str): headers+ headers (dict): headersnoLog (bool): do not log requestpostData (str): HTTP POST datadontMangle (bool): do not mangle@@ -2233,7 +2211,6 @@ class SpiderFoot:Returns:dict: HTTP response"""-if not url:return None@@ -2462,16 +2439,15 @@ class SpiderFoot:self.info(f"Fetched {self.removeUrlCreds(url)} ({len(result['content'] or '')} bytes in {t}s)")return result- def checkDnsWildcard(self, target):- """Check if wildcard DNS is enabled by looking up a random hostname+ def checkDnsWildcard(self, target: str) -> bool:+ """Check if wildcard DNS is enabled by looking up a random subdomain.Args:target (str): TBDReturns:- bool: wildcard DNS+ bool: Domain returns DNS records for any subdomains"""-if not target:return False@@ -2483,7 +2459,7 @@ class SpiderFoot:return True- def googleIterate(self, searchString, opts={}):+ def googleIterate(self, searchString: str, opts: dict = {}) -> dict:"""Request search results from the Google API.Will return a dict:@@ -2501,7 +2477,7 @@ class SpiderFoot:opts (dict): TBDReturns:- dict: TBD+ dict: Search results as {"webSearchUrl": "URL", "urls": [results]}"""search_string = searchString.replace(" ", "%20")@@ -2542,7 +2518,7 @@ class SpiderFoot:"webSearchUrl": f"https://www.google.com/search?q={search_string}&{params}"}- def bingIterate(self, searchString, opts={}):+ def bingIterate(self, searchString: str, opts: dict = {}) -> dict:"""Request search results from the Bing API.Will return a dict:@@ -2561,7 +2537,7 @@ class SpiderFoot:opts (dict): TBDReturns:- dict: TBD+ dict: Search results as {"webSearchUrl": "URL", "urls": [results]}"""search_string = searchString.replace(" ", "%20")