SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sflib: Add guard clauses and docstrings
Brendan Coles055a5f426586
diff --git a/sflib.py b/sflib.pyindex c8da973b..be66fd8c 100644--- a/sflib.py+++ b/sflib.py@@ -730,9 +730,17 @@ class SpiderFoot:# http://abc.com will split to ['http:', '', 'abc.com']return baseurl.split('/')[count].lower()- # Extract the keyword (the domain without the TLD or any subdomains)- # from a domain.def domainKeyword(self, domain, tldList):+ """Extract the keyword (the domain without the TLD or any subdomains) from a domain.++ Args:+ domain (str): The domain to check.+ tldList (str): The list of TLDs based on the Mozilla public list.++ Returns:+ str: The keyword+ """+# Strip off the TLDtld = '.'.join(self.hostDomain(domain.lower(), tldList).split('.')[1:])ret = domain.lower().replace('.' + tld, '')@@ -743,67 +751,137 @@ class SpiderFoot:else:return ret- # Extract the keywords (the domains without the TLD or any subdomains)- # from a list of domains.+ # TODO: remove this functiondef domainKeywords(self, domainList, tldList):+ """Extract the keywords (the domains without the TLD or any subdomains) from a list of domains.++ Wraps the domainKeyword function for people who are too lazy to write a loop.+ Does not validate input. Does not check for duplicates.++ Args:+ domainList (list): The list of domains to check.+ tldList (str): The list of TLDs based on the Mozilla public list.++ Returns:+ list: List of keywords+ """+arr = list()for domain in domainList:- # Strip off the TLD- tld = '.'.join(self.hostDomain(domain.lower(), tldList).split('.')[1:])- ret = domain.lower().replace('.' + tld, '')-- # If the user supplied a domain with a sub-domain, return the second part- if '.' in ret:- arr.append(ret.split('.')[-1])- else:- arr.append(ret)+ arr.append(self.domainKeyword(domain, tldList))self.debug("Keywords: " + str(arr))return arr- # Obtain the domain name for a supplied hostname- # tldList needs to be an array based on the Mozilla public listdef hostDomain(self, hostname, tldList):+ """Obtain the domain name for a supplied hostname.++ Args:+ hostname (str): The hostname to check.+ tldList (str): The list of TLDs based on the Mozilla public list.++ Returns:+ str: The domain name.+ """++ if not tldList:+ return None+ if not hostname:+ return None+ps = PublicSuffixList(tldList)return ps.get_public_suffix(hostname)- # Is the host a valid host (some filenames look like hosts)def validHost(self, hostname, tldList):- # First basic content checks+ """Check if the provided string is a valid hostname with a valid public suffix TLD.++ Args:+ hostname (str): The hostname to check.+ tldList (str): The list of TLDs based on the Mozilla public list.++ Returns:+ bool+ """++ if not tldList:+ return False+ if not hostname:+ return False+if "." not in hostname:return Falseif not re.match("^[a-z0-9-\.]*$", hostname, re.IGNORECASE):return False- # Finally check if it's on a valid TLDps = PublicSuffixList(tldList)sfx = ps.get_public_suffix(hostname, strict=True)return sfx != None- # Given a possible hostname, check if it's a domain name- # By checking whether it rests atop a TLD.- # e.g. www.example.com = False because tld of hostname is com,- # and www.example has a . in it.def isDomain(self, hostname, tldList):+ """Check if the provided hostname string is a valid domain name.++ Given a possible hostname, check if it's a domain name+ By checking whether it rests atop a TLD.+ e.g. www.example.com = False because tld of hostname is com,+ and www.example has a . in it.++ Args:+ hostname (str): The hostname to check.+ tldList (str): The list of TLDs based on the Mozilla public list.++ Returns:+ bool+ """++ if not tldList:+ return False+ if not hostname:+ return False+ps = PublicSuffixList(tldList)suffix = ps.get_public_suffix(hostname)return hostname == suffix- # Simple way to verify IPv4 addresses.def validIP(self, address):+ """Check if the provided string is a valid IPv4 address.++ Args:+ address (str): The IPv4 address to check.++ Returns:+ bool+ """+if not address:return Falsereturn netaddr.valid_ipv4(address)- # Simple way to verify IPv6 addresses.def validIP6(self, address):+ """Check if the provided string is a valid IPv6 address.++ Args:+ address (str): The IPv6 address to check.++ Returns:+ bool+ """+if not address:return Falsereturn netaddr.valid_ipv6(address)# Simple way to verify netblock.def validIpNetwork(self, cidr):+ """Check if the provided string is a valid CIDR netblock.++ Args:+ cidr (str): The netblock to check.++ Returns:+ bool+ """+try:if '/' in str(cidr) and netaddr.IPNetwork(str(cidr)).size > 0:return True@@ -1352,7 +1430,10 @@ class SpiderFoot:# be 'http://xyz.com/abc' with the 'original' attribute set to '/abc'def parseLinks(self, url, data, domains):returnLinks = dict()- urlsRel = []++ if data is None or len(data) == 0:+ self.debug("parseLinks() called with no data to parse.")+ return returnLinksif type(domains) is str:domains = [domains]@@ -1374,9 +1455,7 @@ class SpiderFoot:if proto == None:proto = "http"- if data is None or len(data) == 0:- self.debug("parseLinks() called with no data to parse.")- return None+ urlsRel = []try:for t in list(tags.keys()):@@ -1386,7 +1465,7 @@ class SpiderFoot:urlsRel.append(lnk[tags[t]])except BaseException as e:self.error("Error parsing with BeautifulSoup: " + str(e), False)- return None+ return returnLinks# Loop through all the URLs/links foundfor link in urlsRel: