Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
In case of reload404, load only failed documents
Ian Smirlis53518a91abda
diff --git a/htroot/IndexBrowser_p.java b/htroot/IndexBrowser_p.javaindex 1727e9f9d..6465d10cb 100644--- a/htroot/IndexBrowser_p.java+++ b/htroot/IndexBrowser_p.java@@ -370,6 +370,10 @@ public class IndexBrowser_p {q.append(" AND ").append(CollectionSchema.url_paths_sxt.getSolrFieldName()).append(AbstractSolrConnector.CATCHALL_DTERM);}}++ // if reload errors is requested, limit results to pages with some failreason set+ if(reload404) q.append(" AND ").append(CollectionSchema.failtype_s.getSolrFieldName()).append(AbstractSolrConnector.CATCHALL_DTERM);+final int pageSize = 100;final BlockingQueue<SolrDocument> docs = new ArrayBlockingQueue<>(pageSize);final List<String> queries = new ArrayList<>();@@ -387,9 +391,7 @@ public class IndexBrowser_p {CollectionSchema.references_i.getSolrFieldName(),CollectionSchema.references_internal_i.getSolrFieldName(),CollectionSchema.references_external_i.getSolrFieldName(),- CollectionSchema.references_exthosts_i.getSolrFieldName(),- CollectionSchema.cr_host_chance_d.getSolrFieldName(),- CollectionSchema.cr_host_norm_i.getSolrFieldName()+ CollectionSchema.references_exthosts_i.getSolrFieldName()));solrQueryTask.start();Set<String> storedDocs = new HashSet<String>();@@ -403,7 +405,7 @@ public class IndexBrowser_p {final Set<String> reloadURLCollection = new HashSet<String>();long timeoutList = System.currentTimeMillis() + TIMEOUT;long remainingTime = TIMEOUT;- long timeoutReferences = System.currentTimeMillis() + 6000;+ long timeoutReferences = System.currentTimeMillis() + TIMEOUT;ReferenceReportCache rrCache = sb.index.getReferenceReportCache();try {SolrDocument doc = docs.poll(remainingTime, TimeUnit.MILLISECONDS);@@ -474,6 +476,7 @@ public class IndexBrowser_p {final Map<String, Pattern> cm = new LinkedHashMap<String, Pattern>();for (String collection: reloadURLCollection) cm.put(collection, QueryParams.catchall_pattern);sb.reload(reloadURLs, cm.size() > 0 ? cm : CrawlProfile.collectionParser("user"), false);+}// collect from crawler@@ -486,7 +489,7 @@ public class IndexBrowser_p {for (String u: storedDocs) files.put(u, StoreType.INDEX);for (Map.Entry<String, FailType> e: errorDocs.entrySet()) files.put(e.getKey(), e.getValue() == FailType.fail ? StoreType.FAILED : StoreType.EXCLUDED);for (String u: inboundLinks) if (!files.containsKey(u)) files.put(u, StoreType.LINK);- for (String u: loadingLinks) if (u.startsWith(path) && !files.containsKey(u)) files.put(u, StoreType.LINK);+ if(!reload404) for (String u: loadingLinks) if (u.startsWith(path) && !files.containsKey(u)) files.put(u, StoreType.LINK);ConcurrentLog.info("IndexBrowser_p", "collected " + files.size() + " urls for path " + path);// distinguish files and folders