Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Don't abort when you can't find robots.txt.

Steve4864b7c1931e
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 6a4b7734..624c8731 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -273,9 +273,6 @@ class sfp_spider(SpiderFootPlugin):
if robotsTxt['content'] is not None:
self.sf.debug('robots.txt contents: ' + robotsTxt['content'])
self.robotsRules[targetBase] = self.sf.parseRobotsTxt(robotsTxt['content'])
- else:
- self.sf.error("Unable to fetch robots.txt and you've asked to abide by its contents.")
- return None
if self.checkForStop():
return None