Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Add configurable DHT error URL blocking with retry window and web UI
pr0viehfa61beff3550
diff --git a/defaults/yacy.init b/defaults/yacy.initindex 2274d4d59..0d2fc0dc3 100644--- a/defaults/yacy.init+++ b/defaults/yacy.init@@ -295,7 +295,7 @@ proxyCacheSize = 4096# The compression level for cached content# Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use)-proxyCache.compressionLevel = 9+proxyCache.compressionLevel = 6# Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations# When timeout occurs, loader should fall back to regular remote resource loading@@ -577,6 +577,19 @@ allowReceiveIndex=trueallowReceiveIndex.search=trueindexReceiveBlockBlacklist=true+# Proactive Error-URL Rejection: block distribution of known broken URLs via DHT+# Set to false to disable this feature (opt-out)+indexReceiveBlockErrors=true++# How many days to block error URLs before allowing retry+# Permanent errors (404, 410) are always blocked+# Temporary errors (5xx, timeouts) are blocked for this many days+indexReceiveBlockErrors.retryAfterDays=30++# Comma-separated list of HTTP status codes considered permanent errors (always block)+# Default: 404 (Not Found), 410 (Gone)+indexReceiveBlockErrors.permanentStatus=404,410+# the frequency is the number of links per minute, that the peer allowes# _every_ other peer to send to this peerdefaultWordReceiveFrequency=100@@ -709,7 +722,7 @@ recrawlindex_memprereq=104857640_peerseedcycle_memprereq=419430440_peerseedcycle_loadprereq=2.050_localcrawl_idlesleep=2000-50_localcrawl_busysleep=10+50_localcrawl_busysleep=550_localcrawl_memprereq=2516582450_localcrawl_loadprereq=8.050_localcrawl_isPaused=false@@ -717,13 +730,13 @@ recrawlindex_memprereq=104857655_autocrawl_busysleep=1000055_autocrawl_memprereq=2516582455_autocrawl_loadprereq=8.0-60_remotecrawlloader_idlesleep=4000-60_remotecrawlloader_busysleep=800+60_remotecrawlloader_idlesleep=2000+60_remotecrawlloader_busysleep=20060_remotecrawlloader_memprereq=1258291260_remotecrawlloader_loadprereq=8.060_remotecrawlloader_isPaused=false-62_remotetriggeredcrawl_idlesleep=2000-62_remotetriggeredcrawl_busysleep=200+62_remotetriggeredcrawl_idlesleep=1000+62_remotetriggeredcrawl_busysleep=5062_remotetriggeredcrawl_memprereq=1258291262_remotetriggeredcrawl_loadprereq=8.062_remotetriggeredcrawl_isPaused=false@@ -759,7 +772,7 @@ reindexSolr_loadprereq=16.0# is used to flush the RAM cache, which is the major part of the IO in YaCyperformanceProfile=defaults/yacy.initperformanceSpeed=100-performanceIO=10+performanceIO=15# cleanup-process:# properties for tasks that are performed during cleanup@@ -800,7 +813,13 @@ javastart_priority=10# wordCacheMaxLow/High is the number of word indexes that shall be held in the# ram cache during indexing. If you want to increase indexing speed, increase this# value i.e. up to one million, but increase also the memory limit to a minimum of 2GB-wordCacheMaxCount = 20000+wordCacheMaxCount = 50000++# Maximum number of references per term in the RWI index+# If a term has more references than this value, the oldest references will be removed+# This prevents memory issues with high-frequency terms (stopwords like 'the', 'and', etc.)+# 0 = disabled (no shrinking), 10000 = recommended for standard installations+index.maxReferences = 10000# Specifies if yacy can be used as transparent http proxy.#@@ -1063,7 +1082,7 @@ indexDistribution.maxChunkFails = 1# limit of references per term & blob to the younges of this value# a value of <= 0 disables this feature (no limit)# a value of e.g. 100000 can improve stability and reduce load while searching very popular words-index.maxReferences = 0+index.maxReferences = 10000# Search sequence settings# collection:@@ -1416,24 +1435,24 @@ crawler.userAgent.clienttimeout = 10000# experiments with timeout requeststimeoutrequests = true-# interface decorations-decoration.audio = false-decoration.grafics.linkstructure = true-decoration.hostanalysis = false-decoration.simpleheadernavbar = navbar-default--# ai settings-ai.production_models = []-ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.-ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.-ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n-ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:-ai.shield.allow-nonlocalhost = false-ai.shield.show-chat-link = false-ai.shield.rate.per-minute = 4-ai.shield.rate.per-hour = 60-ai.shield.rate.per-day = 120-ai.shield.limit-all = false-ai.shield.all.per-minute = 16-ai.shield.all.per-hour = 240-ai.shield.all.per-day = 480+# interface decorations+decoration.audio = false+decoration.grafics.linkstructure = true+decoration.hostanalysis = false+decoration.simpleheadernavbar = navbar-default++# ai settings+ai.production_models = []+ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.+ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.+ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n+ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:+ai.shield.allow-nonlocalhost = false+ai.shield.show-chat-link = false+ai.shield.rate.per-minute = 4+ai.shield.rate.per-hour = 60+ai.shield.rate.per-day = 120+ai.shield.limit-all = false+ai.shield.all.per-minute = 16+ai.shield.all.per-hour = 240+ai.shield.all.per-day = 480diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.htmlindex 3edd5ceb7..652899a48 100644--- a/htroot/IndexFederated_p.html+++ b/htroot/IndexFederated_p.html@@ -117,6 +117,15 @@<dt><input type="checkbox" name="core.service.rwi" id="core_service_rwi" #(core.service.rwi.checked)#:: checked="checked"#(/core.service.rwi.checked)# /></dt><dd>support peer-to-peer index transmission (DHT RWI index)</dd>++ <dt class="TableCellDark">Block known error URLs in DHT <input type="checkbox" name="indexReceiveBlockErrors" #(indexReceiveBlockErrors.checked)#:: checked="checked"#(/indexReceiveBlockErrors.checked)# /></dt>+ <dd>Reject URLs/RWIs with known errors from peers. Disable to opt out.</dd>++ <dt class="TableCellDark">Retry after (days)</dt>+ <dd><input type="text" name="indexReceiveBlockErrors.retryAfterDays" value="#[indexReceiveBlockErrors.retryAfterDays]#" size="4" /> for temporary errors; permanent errors stay blocked.</dd>++ <dt class="TableCellDark">Permanent error statuses</dt>+ <dd><input type="text" name="indexReceiveBlockErrors.permanentStatus" value="#[indexReceiveBlockErrors.permanentStatus]#" size="12" /> comma-separated (default: 404,410)</dd><dt><input type="submit" name="setrwi" class="btn btn-primary" value="Set" /></dt><dd></dd></dl>diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.javaindex bbaba2b3a..9b7e8f74c 100644--- a/source/net/yacy/htroot/IndexFederated_p.java+++ b/source/net/yacy/htroot/IndexFederated_p.java@@ -64,6 +64,16 @@ public class IndexFederated_p {final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE);sb.index.connectRWI(wordCacheMaxCount, fileSizeMax);} catch (final IOException e) { ConcurrentLog.logException(e); } // switch on++ // DHT error URL blocking settings+ final boolean blockErrors = post.getBoolean(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS);+ env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, blockErrors);++ final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);+ env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays);++ final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");+ env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent);}if (post != null && post.containsKey("setcitation")) {@@ -222,6 +232,11 @@ public class IndexFederated_p {prop.put("core.service.rwi.checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0);prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);++ // DHT error URL blocking settings+ prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS + ".checked", env.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true) ? 1 : 0);+ prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, env.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30));+ prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, env.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"));prop.put("solr.indexing.solrremote.checked",env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED,SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED_DEFAULT) ? 1 : 0);diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.javaindex 0981ff70e..d02875334 100644--- a/source/net/yacy/htroot/yacy/transferRWI.java+++ b/source/net/yacy/htroot/yacy/transferRWI.java@@ -28,11 +28,15 @@package net.yacy.htroot.yacy;+import java.io.IOException;import java.util.ArrayList;+import java.util.Date;import java.util.HashSet;import java.util.Iterator;import java.util.Set;+import org.apache.solr.common.SolrDocument;+import net.yacy.cora.document.encoding.ASCII;import net.yacy.cora.document.encoding.UTF8;import net.yacy.cora.document.feed.RSSMessage;@@ -55,6 +59,7 @@ import net.yacy.peers.Seed;import net.yacy.repository.Blacklist.BlacklistType;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;+import net.yacy.search.schema.CollectionSchema;import net.yacy.server.serverObjects;import net.yacy.server.serverSwitch;@@ -75,6 +80,7 @@ public final class transferRWI {prop.put("unknownURL", "");prop.put("pause", 60000);String result = "";+ final StringBuilder errorURLs = new StringBuilder(4000);if ((post == null) || (env == null)) {result = "post or env is null!";logWarning(contentType, result);@@ -127,7 +133,6 @@ public final class transferRWI {int pause = 0;result = "ok";final StringBuilder unknownURLs = new StringBuilder(6000);-final double load = Memory.getSystemLoadAverage();final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);if (load > maxload) {@@ -188,6 +193,7 @@ public final class transferRWI {final ArrayList<String> wordhashes = new ArrayList<String>();int received = 0;int blocked = 0;+ int blockedErrors = 0;int count = 0;final Set<String> testids = new HashSet<String>();while (it.hasNext()) {@@ -222,6 +228,75 @@ public final class transferRWI {continue;}+ // reject RWI entries for URLs we already know are broken+ final String urlHashStr = ASCII.String(urlHash);+ final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);+ if (blockErrors && sb.index.fulltext().exists(urlHashStr)) {+ try {+ final SolrDocument errorCheck = sb.index.fulltext().getDefaultConnector().getDocumentById(urlHashStr,+ CollectionSchema.httpstatus_i.getSolrFieldName(), CollectionSchema.failreason_s.getSolrFieldName(),+ CollectionSchema.load_date_dt.getSolrFieldName());+ if (errorCheck != null) {+ final Object httpstatus = errorCheck.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName());+ final Object failreason = errorCheck.getFieldValue(CollectionSchema.failreason_s.getSolrFieldName());+ if (httpstatus != null && failreason != null && failreason.toString().length() > 0) {+ int hs = (httpstatus instanceof Integer) ? (Integer) httpstatus : Integer.parseInt(httpstatus.toString());+ if (hs != 200) {+ boolean shouldBlock = false;++ // Get configuration+ final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);+ final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");+ final Set<Integer> permanentStatus = new HashSet<Integer>();+ for (String s : permanentStatusStr.split(",")) {+ try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}+ }+ final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L;+ final long now = System.currentTimeMillis();++ // Permanent errors (404, 410) - always block+ if (permanentStatus.contains(hs)) {+ shouldBlock = true;+ if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known permanent error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ") from peer " + otherPeerName);+ } else {+ // Temporary errors - check age+ final Object loadDate = errorCheck.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName());+ if (loadDate != null) {+ try {+ final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate :+ new Date(Long.parseLong(loadDate.toString()));+ final long errorAge = now - errorDate.getTime();++ if (errorAge < retryAfterMillis) {+ shouldBlock = true;+ if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known temporary error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName);+ } else {+ if (Network.log.isFine()) Network.log.fine("transferRWI: allowing retry for URL hash '" + urlHashStr + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName);+ }+ } catch (Exception e) {+ // If we can't parse the date, treat as permanent error to be safe+ shouldBlock = true;+ }+ } else {+ // No load_date available - treat as permanent error+ shouldBlock = true;+ }+ }++ if (shouldBlock) {+ errorURLs.append(urlHashStr).append(',');+ blocked++;+ blockedErrors++;+ continue;+ }+ }+ }+ }+ } catch (final IOException e) {+ // ignore Solr errors during error URL check+ }+ }+// learn entrytry {sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry);@@ -251,14 +326,16 @@ public final class transferRWI {unknownURLs.append(UTF8.String(bit.next())).append(',');}if (unknownURLs.length() > 0) { unknownURLs.setLength(unknownURLs.length() - 1); }+ if (errorURLs.length() > 0) { errorURLs.setLength(errorURLs.length() - 1); }+if (wordhashes.isEmpty() || received == 0) {- sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");+ sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs, reporting " + blockedErrors + " error URLs");} else {final String firstHash = wordhashes.get(0);final String lastHash = wordhashes.get(wordhashes.size() - 1);final long avdist = (Distribution.horizontalDHTDistance(firstHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash)) + Distribution.horizontalDHTDistance(lastHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash))) / 2;- sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received+ " URLs from " + otherPeerName);- EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received + " URLs from " + otherPeerName, "", otherPeer.hash));+ sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received+ " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName);+ EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received + " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName, "", otherPeer.hash));}result = "ok";@@ -266,6 +343,7 @@ public final class transferRWI {}prop.put("unknownURL", unknownURLs.toString());+ prop.put("errorURL", errorURLs.toString());prop.put("result", result);prop.put("pause", pause);diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.javaindex f9b95e270..2ca3f533a 100644--- a/source/net/yacy/htroot/yacy/transferURL.java+++ b/source/net/yacy/htroot/yacy/transferURL.java@@ -30,8 +30,11 @@ package net.yacy.htroot.yacy;import java.io.IOException;import java.text.ParseException;+import java.util.Date;import java.util.HashMap;+import java.util.HashSet;import java.util.Map;+import java.util.Set;import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.document.encoding.ASCII;@@ -90,6 +93,7 @@ public final class transferURL {} else {int received = 0;int blocked = 0;+ int blockedErrors = 0;int doublecheck = 0;// read the urls from the other properties and storeString urls;@@ -148,26 +152,72 @@ public final class transferURL {}doublecheck = 0;+ final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);+ final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);+ final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");+ final Set<Integer> permanentStatus = new HashSet<Integer>();+ for (String s : permanentStatusStr.split(",")) {+ try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}+ }+ final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L;+ final long now = System.currentTimeMillis();+for (final String id : lEm.keySet()) {if (sb.index.exists(id)) {doublecheck++;// Check if entry we already have is marked as error - if so, reject incoming replacement- try {- final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id));- if (meta != null && meta.getFieldValue("httpstatus_i") != null) {- final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ?- (Integer) meta.getFieldValue("httpstatus_i") :- Integer.parseInt(meta.getFieldValue("httpstatus_i").toString());- final Object failreason = meta.getFieldValue("failreason_s");- if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) {- if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (known error, httpstatus=" + httpstatus + ") from peer " + otherPeerName);- errorURLs.append(id).append(',');- blocked++;- continue;+ if (blockErrors) {+ try {+ final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id));+ if (meta != null && meta.getFieldValue("httpstatus_i") != null) {+ final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ?+ (Integer) meta.getFieldValue("httpstatus_i") :+ Integer.parseInt(meta.getFieldValue("httpstatus_i").toString());+ final Object failreason = meta.getFieldValue("failreason_s");++ if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) {+ boolean shouldBlock = false;++ // Permanent errors (404, 410) - always block+ if (permanentStatus.contains(httpstatus)) {+ shouldBlock = true;+ if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (permanent error, httpstatus=" + httpstatus + ") from peer " + otherPeerName);+ } else {+ // Temporary errors - check age+ final Object loadDate = meta.getFieldValue("load_date_dt");+ if (loadDate != null) {+ try {+ final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate :+ new Date(Long.parseLong(loadDate.toString()));+ final long errorAge = now - errorDate.getTime();++ if (errorAge < retryAfterMillis) {+ shouldBlock = true;+ if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (temporary error, httpstatus=" + httpstatus + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName);+ } else {+ if (Network.log.isFine()) Network.log.fine("transferURL: allowing retry of URL hash '" + id + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName);+ }+ } catch (Exception e) {+ // If we can't parse the date, treat as permanent error to be safe+ shouldBlock = true;+ }+ } else {+ // No load_date available - treat as permanent error+ shouldBlock = true;+ }+ }++ if (shouldBlock) {+ errorURLs.append(id).append(',');+ blocked++;+ blockedErrors++;+ continue;+ }+ }}+ } catch (final Exception e) {+ // Ignore errors during error status check}- } catch (final Exception e) {- // Ignore errors during error status check}}@@ -190,8 +240,8 @@ public final class transferURL {sb.peers.mySeed().incRU(received);// return rewrite properties- Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");- EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));+ Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs");+ EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs from peer " + otherPeerName, "", otherPeer.hash));if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f);if (doublecheck > 0) {diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.javaindex 91cf24484..3873e15eb 100644--- a/source/net/yacy/search/SwitchboardConstants.java+++ b/source/net/yacy/search/SwitchboardConstants.java@@ -207,6 +207,9 @@ public final class SwitchboardConstants {public static final String INDEX_RECEIVE_ALLOW = "allowReceiveIndex";public static final String INDEX_RECEIVE_ALLOW_SEARCH = "allowReceiveIndex.search";public static final String INDEX_RECEIVE_BLOCK_BLACKLIST = "indexReceiveBlockBlacklist";+ public static final String INDEX_RECEIVE_BLOCK_ERRORS = "indexReceiveBlockErrors";+ public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays";+ public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT = "indexReceiveBlockErrors.permanentStatus";/*** <p><code>public static final String <strong>INDEX_DIST_ALLOW_WHILE_CRAWLING</strong> = "allowDistributeIndexWhileCrawling"</code></p>