Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

CrawlCheck_p.html : also display info about disallowed URLs.

luccioman54cfcc3f5693
diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.java
index 77917c2fc..91d5916bc 100644
--- a/source/net/yacy/crawler/robots/RobotsTxt.java
+++ b/source/net/yacy/crawler/robots/RobotsTxt.java
@@ -393,13 +393,18 @@ public class RobotsTxt {
// try to load the robots
RobotsTxtEntry robotsEntry = getEntry(u, userAgent);
boolean robotsAllowed = robotsEntry == null ? true : !robotsEntry.isDisallowed(u);
- if (robotsAllowed) try {
- Request request = loader.request(u, true, false);
- Response response = loader.load(request, CacheStrategy.NOCACHE, BlacklistType.CRAWLER, userAgent);
- out.put(new CheckEntry(u, robotsEntry, response, null));
- } catch (final IOException e) {
- out.put(new CheckEntry(u, robotsEntry, null, "error response: " + e.getMessage()));
- }
+ if (robotsAllowed) {
+ try {
+ Request request = loader.request(u, true, false);
+ Response response = loader.load(request, CacheStrategy.NOCACHE,
+ BlacklistType.CRAWLER, userAgent);
+ out.put(new CheckEntry(u, robotsEntry, response, null));
+ } catch (final IOException e) {
+ out.put(new CheckEntry(u, robotsEntry, null, "error response: " + e.getMessage()));
+ }
+ } else {
+ out.put(new CheckEntry(u, robotsEntry, null, null));
+ }
}
} catch (InterruptedException e) {}
}