Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Use unredirected robots.txt URL when adding an entry to the table.
luccioman6cec2cdcb5a1
diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.javaindex b97402114..934be322a 100644--- a/source/net/yacy/crawler/robots/RobotsTxt.java+++ b/source/net/yacy/crawler/robots/RobotsTxt.java@@ -198,7 +198,7 @@ public class RobotsTxt {if (response == null) {processOldEntry(robotsTxt4Host, robotsURL, robotsTable);} else {- robotsTxt4Host = processNewEntry(response, agent.robotIDs);+ robotsTxt4Host = processNewEntry(robotsURL, response, agent.robotIDs);}}}@@ -266,7 +266,7 @@ public class RobotsTxt {if (response == null) {processOldEntry(null, robotsURL, robotsTable);} else {- processNewEntry(response, agent.robotIDs);+ processNewEntry(robotsURL, response, agent.robotIDs);}}}@@ -316,11 +316,12 @@ public class RobotsTxt {/*** Process a response to a robots.txt request, create a new robots entry, add it to the robots table then return it.+ * @param robotsURL the initial robots.txt URL (before any eventual redirection). Must not be null.* @param response the response to the requested robots.txt URL. Must not be null.* @param thisAgents the agent identifier(s) used to request the robots.txt URL* @return the new robots entry*/- private RobotsTxtEntry processNewEntry(final Response response, final String[] thisAgents) {+ private RobotsTxtEntry processNewEntry(final DigestURL robotsURL, final Response response, final String[] thisAgents) {final byte[] robotsTxt = response.getContent();//Log.logInfo("RobotsTxt", "robots of " + robotsURL.toNormalform(true, true) + ":\n" + ((robotsTxt == null) ? "null" : UTF8.String(robotsTxt))); // debug TODO removeRobotsTxtParser parserResult;@@ -338,9 +339,14 @@ public class RobotsTxt {// store the data into the robots DBString etag = response.getResponseHeader().containsKey(HeaderFramework.ETAG) ? (response.getResponseHeader().get(HeaderFramework.ETAG)).trim() : null;boolean isBrowserAgent = thisAgents.length == 1 && thisAgents[0].equals("Mozilla");- if (isBrowserAgent) denyPath.clear();+ if (isBrowserAgent) {+ denyPath.clear();+ }+ /* The robotsURL may eventually be redirected (from http to https is common),+ * but we store here the url before any redirection. If would not process this way, the unredirected URL would later+ * never found in the robots table thus needing each time a http load.*/final RobotsTxtEntry robotsTxt4Host = new RobotsTxtEntry(- response.getRequest().url(),+ robotsURL,parserResult.allowList(),denyPath,new Date(),