Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Add DNS/network errors (-1) to permanent error status codes
pr0vieh2744918f9fa1
diff --git a/defaults/yacy.init b/defaults/yacy.initindex 0d2fc0dc3..810f68614 100644--- a/defaults/yacy.init+++ b/defaults/yacy.init@@ -587,8 +587,9 @@ indexReceiveBlockErrors=trueindexReceiveBlockErrors.retryAfterDays=30# Comma-separated list of HTTP status codes considered permanent errors (always block)-# Default: 404 (Not Found), 410 (Gone)-indexReceiveBlockErrors.permanentStatus=404,410+# -1 = DNS/Network errors (UnknownHost, connection failures)+# 404 = Not Found, 410 = Gone+indexReceiveBlockErrors.permanentStatus=404,410,-1# the frequency is the number of links per minute, that the peer allowes# _every_ other peer to send to this peerdiff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.htmlindex 652899a48..c50c95137 100644--- a/htroot/IndexFederated_p.html+++ b/htroot/IndexFederated_p.html@@ -125,7 +125,7 @@<dd><input type="text" name="indexReceiveBlockErrors.retryAfterDays" value="#[indexReceiveBlockErrors.retryAfterDays]#" size="4" /> for temporary errors; permanent errors stay blocked.</dd><dt class="TableCellDark">Permanent error statuses</dt>- <dd><input type="text" name="indexReceiveBlockErrors.permanentStatus" value="#[indexReceiveBlockErrors.permanentStatus]#" size="12" /> comma-separated (default: 404,410)</dd>+ <dd><input type="text" name="indexReceiveBlockErrors.permanentStatus" value="#[indexReceiveBlockErrors.permanentStatus]#" size="15" /> comma-separated (default: 404,410,-1; -1=DNS/network errors)</dd><dt><input type="submit" name="setrwi" class="btn btn-primary" value="Set" /></dt><dd></dd></dl>diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.javaindex 9b7e8f74c..4acf5d996 100644--- a/source/net/yacy/htroot/IndexFederated_p.java+++ b/source/net/yacy/htroot/IndexFederated_p.java@@ -72,7 +72,7 @@ public class IndexFederated_p {final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays);- final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");+ final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1");env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent);}diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.javaindex d02875334..38c0bbbc4 100644--- a/source/net/yacy/htroot/yacy/transferRWI.java+++ b/source/net/yacy/htroot/yacy/transferRWI.java@@ -246,7 +246,7 @@ public final class transferRWI {// Get configurationfinal int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);- final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");+ final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1");final Set<Integer> permanentStatus = new HashSet<Integer>();for (String s : permanentStatusStr.split(",")) {try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.javaindex 2ca3f533a..07af06601 100644--- a/source/net/yacy/htroot/yacy/transferURL.java+++ b/source/net/yacy/htroot/yacy/transferURL.java@@ -154,7 +154,7 @@ public final class transferURL {doublecheck = 0;final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);- final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");+ final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1");final Set<Integer> permanentStatus = new HashSet<Integer>();for (String s : permanentStatusStr.split(",")) {try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.javaindex 3873e15eb..0e391e8ac 100644--- a/source/net/yacy/search/SwitchboardConstants.java+++ b/source/net/yacy/search/SwitchboardConstants.java@@ -209,6 +209,7 @@ public final class SwitchboardConstants {public static final String INDEX_RECEIVE_BLOCK_BLACKLIST = "indexReceiveBlockBlacklist";public static final String INDEX_RECEIVE_BLOCK_ERRORS = "indexReceiveBlockErrors";public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays";+ /** Permanent error HTTP status codes (comma-separated). Default: 404,410,-1 (-1=DNS/network errors) */public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT = "indexReceiveBlockErrors.permanentStatus";/**