Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Improved ErrorCache behavior when switching networks
luccioman3ee4f56c39f4
diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.javaindex 4e272204a..025b5b112 100644--- a/source/net/yacy/crawler/data/CrawlQueues.java+++ b/source/net/yacy/crawler/data/CrawlQueues.java@@ -68,6 +68,7 @@ import net.yacy.search.IndexingQueueEntry;import net.yacy.search.Switchboard;import net.yacy.search.SwitchboardConstants;import net.yacy.search.index.ErrorCache;+import net.yacy.search.index.ErrorCacheFiller;public class CrawlQueues {@@ -95,7 +96,7 @@ public class CrawlQueues {log.config("Opening noticeURL..");this.noticeURL = new NoticedURL(queuePath, sb.getConfigInt("crawler.onDemandLimit", 1000), sb.exceed134217727);log.config("Opening errorURL..");- this.errorURL = new ErrorCache(sb.index.fulltext());+ this.errorURL = new ErrorCache(sb);log.config("Opening delegatedURL..");this.delegatedURL = null;}@@ -117,6 +118,9 @@ public class CrawlQueues {// removed pending requeststhis.workerQueue.clear();this.errorURL.clearCache();+ /* Concurrently refill the error cache with recent errors from the index */+ new ErrorCacheFiller(this.sb, this.errorURL).start();+if (this.remoteCrawlProviderHashes != null) this.remoteCrawlProviderHashes.clear();this.noticeURL.close();this.noticeURL = new NoticedURL(newQueuePath, sb.getConfigInt("crawler.onDemandLimit", 1000), this.sb.exceed134217727);diff --git a/source/net/yacy/search/index/ErrorCache.java b/source/net/yacy/search/index/ErrorCache.javaindex 0cf287396..fedd72f3e 100644--- a/source/net/yacy/search/index/ErrorCache.java+++ b/source/net/yacy/search/index/ErrorCache.java@@ -28,12 +28,8 @@ import java.util.LinkedHashMap;import java.util.Map;import java.util.Set;-import org.apache.solr.client.solrj.SolrQuery;-import org.apache.solr.client.solrj.SolrQuery.SortClause;import org.apache.solr.common.SolrDocument;-import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.common.params.CommonParams;import net.yacy.cora.document.encoding.ASCII;import net.yacy.cora.document.id.DigestURL;@@ -42,6 +38,7 @@ import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;import net.yacy.cora.util.ConcurrentLog;import net.yacy.crawler.data.CrawlProfile;import net.yacy.crawler.robots.RobotsTxt;+import net.yacy.search.Switchboard;import net.yacy.search.schema.CollectionConfiguration;import net.yacy.search.schema.CollectionSchema;@@ -52,51 +49,27 @@ public class ErrorCache {// the class objectprivate final Map<String, CollectionConfiguration.FailDoc> cache;- private final Fulltext fulltext;+ private final Switchboard sb;- public ErrorCache(final Fulltext fulltext) {- this.fulltext = fulltext;+ public ErrorCache(final Switchboard sb) {+ this.sb = sb;this.cache = new LinkedHashMap<String, CollectionConfiguration.FailDoc>();// concurrently fill stack with latest values- new Thread() {- @Override- public void run() {- final SolrQuery params = new SolrQuery();- params.setParam("defType", "edismax");- params.setStart(0);- params.setRows(1000);- params.setFacet(false);- params.setSort(new SortClause(CollectionSchema.load_date_dt.getSolrFieldName(), SolrQuery.ORDER.desc)); // load_date_dt = faildate- params.setFields(CollectionSchema.id.getSolrFieldName());- params.setQuery(CollectionSchema.failreason_s.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);- params.set(CommonParams.DF, CollectionSchema.id.getSolrFieldName()); // DisMaxParams.QF or CommonParams.DF must be given- SolrDocumentList docList;- try {- docList = fulltext.getDefaultConnector().getDocumentListByParams(params);- if (docList != null) for (int i = docList.size() - 1; i >= 0; i--) {- SolrDocument doc = docList.get(i);- String hash = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());- cache.put(hash, null);- }- } catch (IOException e) {- ConcurrentLog.logException(e);- }- }- }.start();+ new ErrorCacheFiller(sb, this).start();}-+public void clearCache() {if (this.cache != null) synchronized (this.cache) {this.cache.clear();}}public void clear() throws IOException {clearCache();- this.fulltext.getDefaultConnector().deleteByQuery(CollectionSchema.failreason_s.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);+ this.sb.index.fulltext().getDefaultConnector().deleteByQuery(CollectionSchema.failreason_s.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);}public void removeHosts(final Set<String> hosthashes) {if (hosthashes == null || hosthashes.size() == 0) return;- this.fulltext.deleteDomainErrors(hosthashes);+ this.sb.index.fulltext().deleteDomainErrors(hosthashes);synchronized (this.cache) {Iterator<String> i = ErrorCache.this.cache.keySet().iterator();while (i.hasNext()) {@@ -105,6 +78,14 @@ public class ErrorCache {}}}++ /**+ * Put a document hash to the internal cache.+ * @param hash document hash.+ */+ public void putHashOnly(String hash) {+ this.cache.put(hash, null);+ }/*** Adds a error document to the Solr index (marked as failed by httpstatus_i <> 200)@@ -129,16 +110,16 @@ public class ErrorCache {url, profile == null ? null : profile.collections(),failCategory.name() + " " + reason, failCategory.failType,httpcode, crawldepth);- if (this.fulltext.getDefaultConnector() != null && failCategory.store && !RobotsTxt.isRobotsURL(url)) {+ if (this.sb.index.fulltext().getDefaultConnector() != null && failCategory.store && !RobotsTxt.isRobotsURL(url)) {// send the error to solrtry {// do not overwrite error reports with error reports- SolrDocument olddoc = this.fulltext.getDefaultConnector().getDocumentById(ASCII.String(failDoc.getDigestURL().hash()), CollectionSchema.httpstatus_i.getSolrFieldName());+ SolrDocument olddoc = this.sb.index.fulltext().getDefaultConnector().getDocumentById(ASCII.String(failDoc.getDigestURL().hash()), CollectionSchema.httpstatus_i.getSolrFieldName());if (olddoc == null ||olddoc.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName()) == null ||((Integer) olddoc.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName())) == 200) {- SolrInputDocument errorDoc = failDoc.toSolr(this.fulltext.getDefaultConfiguration());- this.fulltext.getDefaultConnector().add(errorDoc);+ SolrInputDocument errorDoc = failDoc.toSolr(this.sb.index.fulltext().getDefaultConfiguration());+ this.sb.index.fulltext().getDefaultConnector().add(errorDoc);}} catch (final IOException e) {ConcurrentLog.warn("SOLR", "failed to send error " + url.toNormalform(true) + " to solr: " + e.getMessage());@@ -174,7 +155,7 @@ public class ErrorCache {String hash = entry.getKey();CollectionConfiguration.FailDoc failDoc = entry.getValue();if (failDoc == null) {- SolrDocument doc = this.fulltext.getDefaultConnector().getDocumentById(hash);+ SolrDocument doc = this.sb.index.fulltext().getDefaultConnector().getDocumentById(hash);if (doc != null) failDoc = new CollectionConfiguration.FailDoc(doc);}if (failDoc != null) l.add(failDoc);@@ -193,7 +174,7 @@ public class ErrorCache {}if (failDoc != null) return failDoc;try {- final SolrDocument doc = this.fulltext.getDefaultConnector().getDocumentById(urlhash);+ final SolrDocument doc = this.sb.index.fulltext().getDefaultConnector().getDocumentById(urlhash);if (doc == null) return null;Object failreason = doc.getFieldValue(CollectionSchema.failreason_s.getSolrFieldName());if (failreason == null || failreason.toString().length() == 0) return null;@@ -207,7 +188,7 @@ public class ErrorCache {String urlHashString = ASCII.String(urlHash);try {// load the fail reason, if exists- final SolrDocument doc = this.fulltext.getDefaultConnector().getDocumentById(urlHashString, CollectionSchema.failreason_s.getSolrFieldName());+ final SolrDocument doc = this.sb.index.fulltext().getDefaultConnector().getDocumentById(urlHashString, CollectionSchema.failreason_s.getSolrFieldName());if (doc == null) return false;// check if the document contains a value in the field CollectionSchema.failreason_sdiff --git a/source/net/yacy/search/index/ErrorCacheFiller.java b/source/net/yacy/search/index/ErrorCacheFiller.javanew file mode 100644index 000000000..a4ba0a262--- /dev/null+++ b/source/net/yacy/search/index/ErrorCacheFiller.java@@ -0,0 +1,88 @@+/**+ * ErrorCache+ * Copyright 2016 by luccioman+ *+ * This library is free software; you can redistribute it and/or+ * modify it under the terms of the GNU Lesser General Public+ * License as published by the Free Software Foundation; either+ * version 2.1 of the License, or (at your option) any later version.+ *+ * This library is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU+ * Lesser General Public License for more details.+ *+ * You should have received a copy of the GNU Lesser General Public License+ * along with this program in the file lgpl21.txt+ * If not, see <http://www.gnu.org/licenses/>.+ */++package net.yacy.search.index;++import java.io.IOException;++import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrQuery.SortClause;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrDocumentList;+import org.apache.solr.common.params.CommonParams;++import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;+import net.yacy.cora.util.ConcurrentLog;+import net.yacy.search.Switchboard;+import net.yacy.search.schema.CollectionSchema;++/**+ * A task to concurrently fill the ErrorCache from the index+ * @author luccioman+ *+ */+public class ErrorCacheFiller extends Thread {++ /** Switchboard instance */+ private Switchboard sb;++ /** The cache to fill */+ private ErrorCache cache;++ /**+ * Constructor : this prepares the concurrent task+ * @param sb switchboard instance. Must not be null.+ * @param cache error cache to fill. Must not be null.+ */+ public ErrorCacheFiller(Switchboard sb, ErrorCache cache) {+ if(sb == null || cache == null) {+ throw new IllegalArgumentException("Unexpected null parameters");+ }+ this.sb = sb;+ this.cache = cache;+ }++ /**+ * Fills the error cache with recently failed document hashes found in the index+ */+ @Override+ public void run() {+ final SolrQuery params = new SolrQuery();+ params.setParam("defType", "edismax");+ params.setStart(0);+ params.setRows(1000);+ params.setFacet(false);+ params.setSort(new SortClause(CollectionSchema.load_date_dt.getSolrFieldName(), SolrQuery.ORDER.desc)); // load_date_dt = faildate+ params.setFields(CollectionSchema.id.getSolrFieldName());+ params.setQuery(CollectionSchema.failreason_s.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);+ params.set(CommonParams.DF, CollectionSchema.id.getSolrFieldName()); // DisMaxParams.QF or CommonParams.DF must be given+ SolrDocumentList docList;+ try {+ docList = this.sb.index.fulltext().getDefaultConnector().getDocumentListByParams(params);+ if (docList != null) for (int i = docList.size() - 1; i >= 0; i--) {+ SolrDocument doc = docList.get(i);+ String hash = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());+ cache.putHashOnly(hash);+ }+ } catch (IOException e) {+ ConcurrentLog.logException(e);+ }+ }++}