Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Enabled displaying results after 14th page for local search queries.
lucciomanc25e48e969f1
diff --git a/htroot/js/yacysearch.js b/htroot/js/yacysearch.jsindex 2c7366081..0b754d23a 100644--- a/htroot/js/yacysearch.js+++ b/htroot/js/yacysearch.js@@ -39,11 +39,72 @@ function fadeOutBar() {document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;backgroundColor:transparent;");}-function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase) {- if (totalcount == 0) return;- if (offset >= 0) document.getElementById("offset").innerHTML = offset;- if (offset >= 0) document.getElementById("startRecord").setAttribute('value', offset - 1);- if (itemscount >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;+/**+ * @returns pagination buttons+ */+function renderPaginationButtons(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {+ var resnav = "<ul class=\"pagination\">";+ var thispage = Math.floor(offset / itemsperpage);+ var firstPage = thispage - (thispage % 10);+ if (thispage == 0) {+ resnav += "<li class=\"disabled\"><a href=\"#\">«</a></li>";+ } else {+ resnav += "<li><a id=\"prevpage\" href=\"";+ resnav += (navurlbase + "&startRecord=" + ((thispage - 1) * itemsperpage));+ resnav += "\">«</a></li>";+ }++ var totalPagesNb = Math.floor(1 + ((totalcount - 1) / itemsperpage));+ var numberofpages = Math.min(10, totalPagesNb - firstPage);+ if (!numberofpages) numberofpages = 10;+ for (i = firstPage; i < (firstPage + numberofpages); i++) {+ if (i == thispage) {+ resnav += "<li class=\"active\"><a href=\"#\">";+ resnav += (i + 1);+ resnav += "</a></li>";+ } else {+ resnav += "<li><a href=\"";+ resnav += (navurlbase + "&startRecord=" + (i * itemsperpage));+ resnav += "\">" + (i + 1) + "</a></li>";+ }+ }+ if ((localQuery && thispage >= (totalPagesNb - 1)) || (!localQuery && thispage >= (numberofpages - 1))) {+ resnav += "<li class=\"disabled\"><a href=\"#\">»</a></li>";+ } else {+ resnav += "<li><a id=\"nextpage\" href=\"";+ resnav += (navurlbase + "&startRecord=" + ((thispage + 1) * itemsperpage));+ resnav += "\">»</a>";+ }+ resnav += "</ul>";+ return resnav;+}++/**+ * Parses a string representing an integer value+ * @param strIntValue formatted string+ * @returns the number value or undefined when the string is undefined, or NaN when the string is not a number+ */+function parseFormattedInt(strIntValue) {+ var inValue;+ if(strIntValue != null && strIntValue.replace != null) {+ /* Remove thousands separator and try to parse as integer */+ intValue = parseInt(strIntValue.replace(/[\.\,]/g,''))+ }+ return intValue;+}++function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {+ var totalcountIntValue = parseFormattedInt(totalcount);+ var offsetIntValue = parseFormattedInt(offset);+ var itemscountIntValue = parseFormattedInt(itemscount);+ var itemsperpageIntValue = parseFormattedInt(itemsperpage);++ if (totalcountIntValue == 0) {+ return;+ }+ if (offsetIntValue >= 0) document.getElementById("offset").innerHTML = offset;+ if (offsetIntValue >= 0) document.getElementById("startRecord").setAttribute('value', offsetIntValue - 1);+ if (itemscountIntValue >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;document.getElementById("totalcount").firstChild.nodeValue = totalcount;if (document.getElementById("localResourceSize") != null) document.getElementById("localResourceSize").firstChild.nodeValue = localResourceSize;if (document.getElementById("remoteResourceSize") != null) document.getElementById("remoteResourceSize").firstChild.nodeValue = remoteResourceSize;@@ -52,7 +113,7 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS// compose page navigationif (document.getElementById("progressbar").getAttribute('class') != "progress-bar progress-bar-success") {- var percent = 100 * (itemscount - offset + 1) / itemsperpage;+ var percent = 100 * (itemscountIntValue - offsetIntValue + 1) / itemsperpageIntValue;if (percent == 100) {document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;");document.getElementById("progressbar").setAttribute('class',"progress-bar progress-bar-success");@@ -62,38 +123,9 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS}var resnavElement = document.getElementById("resNav");if (resnavElement != null) {- var resnav = "<ul class=\"pagination\">";- thispage = Math.floor(offset / itemsperpage);- if (thispage == 0) {- resnav += "<li class=\"disabled\"><a href=\"#\">«</a></li>";- } else {- resnav += "<li><a id=\"prevpage\" href=\"";- resnav += (navurlbase + "&startRecord=" + ((thispage - 1) * itemsperpage));- resnav += "\">«</a></li>";- }-- numberofpages = Math.floor(Math.min(9, 1 + ((totalcount.replace(/\./g,'') - 1) / itemsperpage)));- if (!numberofpages) numberofpages = 9;- for (i = 0; i < numberofpages; i++) {- if (i == thispage) {- resnav += "<li class=\"active\"><a href=\"#\">";- resnav += (i + 1);- resnav += "</a></li>";- } else {- resnav += "<li><a href=\"";- resnav += (navurlbase + "&startRecord=" + (i * itemsperpage));- resnav += "\">" + (i + 1) + "</a></li>";- }- }- if (thispage >= numberofpages) {- resnav += "<li><a href=\"#\">»</a></li>";- } else {- resnav += "<li><a id=\"nextpage\" href=\"";- resnav += (navurlbase + "&startRecord=" + ((thispage + 1) * itemsperpage));- resnav += "\">»</a>";- }- resnav += "</ul>";- resnavElement.innerHTML = resnav;+ resnavElement.innerHTML = renderPaginationButtons(offsetIntValue, itemscountIntValue, itemsperpageIntValue,+ totalcountIntValue, parseFormattedInt(localResourceSize), parseFormattedInt(remoteResourceSize), parseFormattedInt(remoteIndexCount),+ parseFormattedInt(remotePeerCount), navurlbase, localQuery);}}diff --git a/htroot/yacysearch.html b/htroot/yacysearch.htmlindex 180a7f01f..a7f2aa476 100644--- a/htroot/yacysearch.html+++ b/htroot/yacysearch.html@@ -222,7 +222,8 @@ function latestinfo() {self.xmlHttpReq.onreadystatechange = function() {if (self.xmlHttpReq.readyState == 4) {var rsp = eval("(" + self.xmlHttpReq.responseText + ")");- statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase);++ statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase, #[localQuery]#);}};self.xmlHttpReq.send(null);diff --git a/htroot/yacysearch.java b/htroot/yacysearch.javaindex 56beb3c6e..7c2f850ef 100644--- a/htroot/yacysearch.java+++ b/htroot/yacysearch.java@@ -171,6 +171,7 @@ public class yacysearch {prop.put("indexof", "off");prop.put("constraint", "");prop.put("depth", "0");+ prop.put("localQuery", "0");prop.put("search.verify",(post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh"));@@ -849,6 +850,7 @@ public class yacysearch {}prop.put("depth", "0");+ prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");}prop.put("focus", focus ? 1 : 0); // focus search fielddiff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.javaindex 271c9c4a5..8874b2e33 100644--- a/htroot/yacysearchitem.java+++ b/htroot/yacysearchitem.java@@ -101,6 +101,7 @@ public class yacysearchitem {prop.put("references", "0");prop.put("rssreferences", "0");prop.put("dynamic", "0");+ prop.put("localQuery", "0");// find search eventfinal SearchEvent theSearch = SearchEventCache.getEvent(eventID);@@ -119,6 +120,7 @@ public class yacysearchitem {prop.put("remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));prop.put("remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));prop.put("navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());+ prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.javaindex 3d28b9df2..263fc2c8d 100644--- a/source/net/yacy/search/query/QueryParams.java+++ b/source/net/yacy/search/query/QueryParams.java@@ -187,7 +187,13 @@ public final class QueryParams {this.contentdom = contentdom;this.timezoneOffset = timezoneOffset;this.itemsPerPage = Math.min((specialRights) ? 10000 : 1000, itemsPerPage);- this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));+ if(domType == Searchdom.LOCAL) {+ /* No offset restriction on local index only requests, as only itemsPerPage will be loaded */+ this.offset = Math.max(0, offset);+ } else {+ /* Offset has to be limited on requests mixing local and remote results, because all results before offset are loaded */+ this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));+ }try {this.urlMaskString = urlMask;// solr doesn't like slashes, backslashes or doublepoints; remove them // urlmask = ".*\\." + ft + "(\\?.*)?";diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.javaindex 7cc76d4c0..af63bd2e0 100644--- a/source/net/yacy/search/query/SearchEvent.java+++ b/source/net/yacy/search/query/SearchEvent.java@@ -142,6 +142,8 @@ public final class SearchEvent {private byte[] IAmaxcounthash, IAneardhthash;public Thread rwiProcess;public Thread localsolrsearch;+ /** Offset of the next local Solr index request+ * Example : last local request with offset=10 and itemsPerPage=20, sets this attribute to 30. */private int localsolroffset;private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasonspublic final ScoreMap<String> locationNavigator; // a counter for the appearance of location coordinates@@ -323,9 +325,9 @@ public final class SearchEvent {// start a local solr searchif (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {- this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);+ this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), this.query.offset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);}- this.localsolroffset = this.query.itemsPerPage;+ this.localsolroffset = this.query.offset + this.query.itemsPerPage;// start a local RWI search concurrentlythis.rwiProcess = null;@@ -1576,31 +1578,68 @@ public final class SearchEvent {// (happens if a search pages is accessed a second time)final long finishTime = timeout == Long.MAX_VALUE ? Long.MAX_VALUE : System.currentTimeMillis() + timeout;EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "started, item = " + item + ", available = " + this.getResultCount(), 0, 0), false);+// wait until a local solr is finished, we must do that to be able to check if we need more- if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join(100);} catch (final InterruptedException e) {}}- if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}- if (item >= this.localsolroffset && this.local_solr_stored.get() >= item) {- // load remaining solr results now+ if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {+ try {+ this.localsolrsearch.join(100);+ } catch (final InterruptedException e) {+ log.warn("Wait for local solr search was interrupted.");+ }+ }+ if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {+ try {+ this.localsolrsearch.join();+ } catch (final InterruptedException e) {+ log.warn("Wait for local solr search was interrupted.");+ }+ }+ if (this.remote && item >= this.localsolroffset && this.local_solr_stored.get() >= item) {+ /* Request mixing remote and local Solr results : load remaining local solr results now.+ * For local only search, a new SearchEvent should be created, starting directly at the requested offset,+ * thus allowing to handle last pages of large resultsets+ */int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}- if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {- this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, 0, Switchboard.urlBlacklist);- }- this.localsolroffset += nextitems;+ if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {+ this.localsolrsearch = RemoteSearch.solrRemoteSearch(this,+ this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image),+ this.localsolroffset, nextitems, null /* this peer */, 0, Switchboard.urlBlacklist);+ }+ this.localsolroffset += nextitems;}// now pull results as long as needed and as long as possible- if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) try {Thread.sleep(100);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}- while ( this.resultList.sizeAvailable() <= item &&+ if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) {+ try {+ Thread.sleep(100);+ } catch (final InterruptedException e) {+ log.warn("Remote search results wait was interrupted.");+ }+ }++ final int resultListIndex;+ if (this.remote) {+ resultListIndex = item;+ } else {+ resultListIndex = item - (this.localsolroffset - this.query.itemsPerPage);+ }+ while ( this.resultList.sizeAvailable() <= resultListIndex &&(this.rwiQueueSize() > 0 || this.nodeStack.sizeQueue() > 0 ||(!this.feedingIsFinished() && System.currentTimeMillis() < finishTime))) {- if (!drainStacksToResult()) try {Thread.sleep(10);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}+ if (!drainStacksToResult()) {+ try {+ Thread.sleep(10);+ } catch (final InterruptedException e) {+ log.warn("Search results wait was interrupted.");+ }+ }}// check if we have a success- if (this.resultList.sizeAvailable() > item) {+ if (this.resultList.sizeAvailable() > resultListIndex) {// we have the wanted result already in the result array .. return that- final URIMetadataNode re = this.resultList.element(item).getElement();+ final URIMetadataNode re = this.resultList.element(resultListIndex).getElement();EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "fetched, item = " + item + ", available = " + this.getResultCount() + ": " + re.urlstring(), 0, 0), false);/*diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.javaindex 0f8bb73fc..b8d9d4080 100644--- a/source/net/yacy/search/query/SearchEventCache.java+++ b/source/net/yacy/search/query/SearchEventCache.java@@ -155,11 +155,24 @@ public class SearchEventCache {event = null;} else {if (event != null) {- //re-new the event time for this event, so it is not deleted next time too early- event.resetEventTime();- // replace the current result offset- event.query.offset = query.offset;- event.query.itemsPerPage = query.itemsPerPage;+ if(query.isLocal()) {+ /* Searching the local index only : we do not reuse the cached event each time the page size or offset changes.+ * This allow to request last result pages of large result sets (larger than SearchEvent.max_results_node)+ * without the need to retrieve all the beginning pages */+ if(event.query.offset != query.offset || event.query.itemsPerPage != query.itemsPerPage) {+ synchronized (lastEvents) {+ lastEvents.remove(id);+ }+ cacheDelete++;+ event = null;+ }+ } else {+ //re-new the event time for this event, so it is not deleted next time too early+ event.resetEventTime();+ // replace the current result offset+ event.query.offset = query.offset;+ event.query.itemsPerPage = query.itemsPerPage;+ }}}if (event == null) {