Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
CrawlCheck_p.html : also display info about disallowed URLs.
luccioman54cfcc3f5693
diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.javaindex 77917c2fc..91d5916bc 100644--- a/source/net/yacy/crawler/robots/RobotsTxt.java+++ b/source/net/yacy/crawler/robots/RobotsTxt.java@@ -393,13 +393,18 @@ public class RobotsTxt {// try to load the robotsRobotsTxtEntry robotsEntry = getEntry(u, userAgent);boolean robotsAllowed = robotsEntry == null ? true : !robotsEntry.isDisallowed(u);- if (robotsAllowed) try {- Request request = loader.request(u, true, false);- Response response = loader.load(request, CacheStrategy.NOCACHE, BlacklistType.CRAWLER, userAgent);- out.put(new CheckEntry(u, robotsEntry, response, null));- } catch (final IOException e) {- out.put(new CheckEntry(u, robotsEntry, null, "error response: " + e.getMessage()));- }+ if (robotsAllowed) {+ try {+ Request request = loader.request(u, true, false);+ Response response = loader.load(request, CacheStrategy.NOCACHE,+ BlacklistType.CRAWLER, userAgent);+ out.put(new CheckEntry(u, robotsEntry, response, null));+ } catch (final IOException e) {+ out.put(new CheckEntry(u, robotsEntry, null, "error response: " + e.getMessage()));+ }+ } else {+ out.put(new CheckEntry(u, robotsEntry, null, null));+ }}} catch (InterruptedException e) {}}