Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Fixed stored URL in web cache when redirection(s) occurs.
luccioman7baa99f26f75
diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.javaindex 0162297e0..5a11aa24b 100644--- a/source/net/yacy/crawler/retrieval/Response.java+++ b/source/net/yacy/crawler/retrieval/Response.java@@ -257,6 +257,11 @@ public class Response {return this.request.name();}+ /**+ * @return the requested URL that produced this response. When redirection(s)+ * occurred, this is not the initial URL, but the last redirection+ * target.+ */public DigestURL url() {return this.request.url();}diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.javaindex 897ae9f5c..03feb047d 100644--- a/source/net/yacy/repository/LoaderDispatcher.java+++ b/source/net/yacy/repository/LoaderDispatcher.java@@ -267,7 +267,11 @@ public final class LoaderDispatcher {final String storeError = response.shallStoreCacheForCrawler();if (storeError == null) {try {- Cache.store(url, response.getResponseHeader(), response.getContent());+ /* Important : we associate here the loaded content with the URL response.url().+ * On eventual redirection(s), response.url() provides the last redirection location.+ * If instead we associated content with the initial url (beginning of the redirection(s) chain),+ * the parsers would then have a wrong base URL when following links with relative URLs. */+ Cache.store(response.url(), response.getResponseHeader(), response.getContent());} catch (final IOException e) {LoaderDispatcher.log.warn("cannot write " + response.url() + " to Cache (3): " + e.getMessage(), e);}