Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Enabled displaying results after 14th page for local search queries.

lucciomanc25e48e969f1
diff --git a/htroot/js/yacysearch.js b/htroot/js/yacysearch.js
index 2c7366081..0b754d23a 100644
--- a/htroot/js/yacysearch.js
+++ b/htroot/js/yacysearch.js
@@ -39,11 +39,72 @@ function fadeOutBar() {
document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;backgroundColor:transparent;");
}
-function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase) {
- if (totalcount == 0) return;
- if (offset >= 0) document.getElementById("offset").innerHTML = offset;
- if (offset >= 0) document.getElementById("startRecord").setAttribute('value', offset - 1);
- if (itemscount >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;
+/**
+ * @returns pagination buttons
+ */
+function renderPaginationButtons(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {
+ var resnav = "<ul class=\"pagination\">";
+ var thispage = Math.floor(offset / itemsperpage);
+ var firstPage = thispage - (thispage % 10);
+ if (thispage == 0) {
+ resnav += "<li class=\"disabled\"><a href=\"#\">&laquo;</a></li>";
+ } else {
+ resnav += "<li><a id=\"prevpage\" href=\"";
+ resnav += (navurlbase + "&amp;startRecord=" + ((thispage - 1) * itemsperpage));
+ resnav += "\">&laquo;</a></li>";
+ }
+
+ var totalPagesNb = Math.floor(1 + ((totalcount - 1) / itemsperpage));
+ var numberofpages = Math.min(10, totalPagesNb - firstPage);
+ if (!numberofpages) numberofpages = 10;
+ for (i = firstPage; i < (firstPage + numberofpages); i++) {
+ if (i == thispage) {
+ resnav += "<li class=\"active\"><a href=\"#\">";
+ resnav += (i + 1);
+ resnav += "</a></li>";
+ } else {
+ resnav += "<li><a href=\"";
+ resnav += (navurlbase + "&amp;startRecord=" + (i * itemsperpage));
+ resnav += "\">" + (i + 1) + "</a></li>";
+ }
+ }
+ if ((localQuery && thispage >= (totalPagesNb - 1)) || (!localQuery && thispage >= (numberofpages - 1))) {
+ resnav += "<li class=\"disabled\"><a href=\"#\">&raquo;</a></li>";
+ } else {
+ resnav += "<li><a id=\"nextpage\" href=\"";
+ resnav += (navurlbase + "&amp;startRecord=" + ((thispage + 1) * itemsperpage));
+ resnav += "\">&raquo;</a>";
+ }
+ resnav += "</ul>";
+ return resnav;
+}
+
+/**
+ * Parses a string representing an integer value
+ * @param strIntValue formatted string
+ * @returns the number value or undefined when the string is undefined, or NaN when the string is not a number
+ */
+function parseFormattedInt(strIntValue) {
+ var inValue;
+ if(strIntValue != null && strIntValue.replace != null) {
+ /* Remove thousands separator and try to parse as integer */
+ intValue = parseInt(strIntValue.replace(/[\.\,]/g,''))
+ }
+ return intValue;
+}
+
+function statistics(offset, itemscount, itemsperpage, totalcount, localResourceSize, remoteResourceSize, remoteIndexCount, remotePeerCount, navurlbase, localQuery) {
+ var totalcountIntValue = parseFormattedInt(totalcount);
+ var offsetIntValue = parseFormattedInt(offset);
+ var itemscountIntValue = parseFormattedInt(itemscount);
+ var itemsperpageIntValue = parseFormattedInt(itemsperpage);
+
+ if (totalcountIntValue == 0) {
+ return;
+ }
+ if (offsetIntValue >= 0) document.getElementById("offset").innerHTML = offset;
+ if (offsetIntValue >= 0) document.getElementById("startRecord").setAttribute('value', offsetIntValue - 1);
+ if (itemscountIntValue >= 0) document.getElementById("itemscount").firstChild.nodeValue = itemscount;
document.getElementById("totalcount").firstChild.nodeValue = totalcount;
if (document.getElementById("localResourceSize") != null) document.getElementById("localResourceSize").firstChild.nodeValue = localResourceSize;
if (document.getElementById("remoteResourceSize") != null) document.getElementById("remoteResourceSize").firstChild.nodeValue = remoteResourceSize;
@@ -52,7 +113,7 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS
// compose page navigation
if (document.getElementById("progressbar").getAttribute('class') != "progress-bar progress-bar-success") {
- var percent = 100 * (itemscount - offset + 1) / itemsperpage;
+ var percent = 100 * (itemscountIntValue - offsetIntValue + 1) / itemsperpageIntValue;
if (percent == 100) {
document.getElementById("progressbar").setAttribute('style',"transition:transform 0s;-webkit-transition:-webkit-transform 0s;");
document.getElementById("progressbar").setAttribute('class',"progress-bar progress-bar-success");
@@ -62,38 +123,9 @@ function statistics(offset, itemscount, itemsperpage, totalcount, localResourceS
}
var resnavElement = document.getElementById("resNav");
if (resnavElement != null) {
- var resnav = "<ul class=\"pagination\">";
- thispage = Math.floor(offset / itemsperpage);
- if (thispage == 0) {
- resnav += "<li class=\"disabled\"><a href=\"#\">&laquo;</a></li>";
- } else {
- resnav += "<li><a id=\"prevpage\" href=\"";
- resnav += (navurlbase + "&amp;startRecord=" + ((thispage - 1) * itemsperpage));
- resnav += "\">&laquo;</a></li>";
- }
-
- numberofpages = Math.floor(Math.min(9, 1 + ((totalcount.replace(/\./g,'') - 1) / itemsperpage)));
- if (!numberofpages) numberofpages = 9;
- for (i = 0; i < numberofpages; i++) {
- if (i == thispage) {
- resnav += "<li class=\"active\"><a href=\"#\">";
- resnav += (i + 1);
- resnav += "</a></li>";
- } else {
- resnav += "<li><a href=\"";
- resnav += (navurlbase + "&amp;startRecord=" + (i * itemsperpage));
- resnav += "\">" + (i + 1) + "</a></li>";
- }
- }
- if (thispage >= numberofpages) {
- resnav += "<li><a href=\"#\">&raquo;</a></li>";
- } else {
- resnav += "<li><a id=\"nextpage\" href=\"";
- resnav += (navurlbase + "&amp;startRecord=" + ((thispage + 1) * itemsperpage));
- resnav += "\">&raquo;</a>";
- }
- resnav += "</ul>";
- resnavElement.innerHTML = resnav;
+ resnavElement.innerHTML = renderPaginationButtons(offsetIntValue, itemscountIntValue, itemsperpageIntValue,
+ totalcountIntValue, parseFormattedInt(localResourceSize), parseFormattedInt(remoteResourceSize), parseFormattedInt(remoteIndexCount),
+ parseFormattedInt(remotePeerCount), navurlbase, localQuery);
}
}
diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html
index 180a7f01f..a7f2aa476 100644
--- a/htroot/yacysearch.html
+++ b/htroot/yacysearch.html
@@ -222,7 +222,8 @@ function latestinfo() {
self.xmlHttpReq.onreadystatechange = function() {
if (self.xmlHttpReq.readyState == 4) {
var rsp = eval("(" + self.xmlHttpReq.responseText + ")");
- statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase);
+
+ statistics(rsp.offset, rsp.itemscount, rsp.itemsperpage, rsp.totalcount, rsp.localResourceSize, rsp.remoteResourceSize, rsp.remoteIndexCount, rsp.remotePeerCount, rsp.navurlBase, #[localQuery]#);
}
};
self.xmlHttpReq.send(null);
diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java
index 56beb3c6e..7c2f850ef 100644
--- a/htroot/yacysearch.java
+++ b/htroot/yacysearch.java
@@ -171,6 +171,7 @@ public class yacysearch {
prop.put("indexof", "off");
prop.put("constraint", "");
prop.put("depth", "0");
+ prop.put("localQuery", "0");
prop.put(
"search.verify",
(post == null) ? sb.getConfig("search.verify", "iffresh") : post.get("verify", "iffresh"));
@@ -849,6 +850,7 @@ public class yacysearch {
}
prop.put("depth", "0");
+ prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
}
prop.put("focus", focus ? 1 : 0); // focus search field
diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java
index 271c9c4a5..8874b2e33 100644
--- a/htroot/yacysearchitem.java
+++ b/htroot/yacysearchitem.java
@@ -101,6 +101,7 @@ public class yacysearchitem {
prop.put("references", "0");
prop.put("rssreferences", "0");
prop.put("dynamic", "0");
+ prop.put("localQuery", "0");
// find search event
final SearchEvent theSearch = SearchEventCache.getEvent(eventID);
@@ -119,6 +120,7 @@ public class yacysearchitem {
prop.put("remoteIndexCount", Formatter.number(theSearch.remote_rwi_available.get() + theSearch.remote_solr_available.get(), true));
prop.put("remotePeerCount", Formatter.number(theSearch.remote_rwi_peerCount.get() + theSearch.remote_solr_peerCount.get(), true));
prop.put("navurlBase", QueryParams.navurlBase(RequestHeader.FileType.HTML, theSearch.query, null, false).toString());
+ prop.put("localQuery", theSearch.query.isLocal() ? "1" : "0");
final String target_special_pattern = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "");
long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000);
diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java
index 3d28b9df2..263fc2c8d 100644
--- a/source/net/yacy/search/query/QueryParams.java
+++ b/source/net/yacy/search/query/QueryParams.java
@@ -187,7 +187,13 @@ public final class QueryParams {
this.contentdom = contentdom;
this.timezoneOffset = timezoneOffset;
this.itemsPerPage = Math.min((specialRights) ? 10000 : 1000, itemsPerPage);
- this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
+ if(domType == Searchdom.LOCAL) {
+ /* No offset restriction on local index only requests, as only itemsPerPage will be loaded */
+ this.offset = Math.max(0, offset);
+ } else {
+ /* Offset has to be limited on requests mixing local and remote results, because all results before offset are loaded */
+ this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
+ }
try {
this.urlMaskString = urlMask;
// solr doesn't like slashes, backslashes or doublepoints; remove them // urlmask = ".*\\." + ft + "(\\?.*)?";
diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java
index 7cc76d4c0..af63bd2e0 100644
--- a/source/net/yacy/search/query/SearchEvent.java
+++ b/source/net/yacy/search/query/SearchEvent.java
@@ -142,6 +142,8 @@ public final class SearchEvent {
private byte[] IAmaxcounthash, IAneardhthash;
public Thread rwiProcess;
public Thread localsolrsearch;
+ /** Offset of the next local Solr index request
+ * Example : last local request with offset=10 and itemsPerPage=20, sets this attribute to 30. */
private int localsolroffset;
private final AtomicInteger expectedRemoteReferences, maxExpectedRemoteReferences; // counter for referenced that had been sorted out for other reasons
public final ScoreMap<String> locationNavigator; // a counter for the appearance of location coordinates
@@ -323,9 +325,9 @@ public final class SearchEvent {
// start a local solr search
if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
- this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), 0, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
+ this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, true, this.excludeintext_image), this.query.offset, this.query.itemsPerPage, null /*this peer*/, 0, Switchboard.urlBlacklist);
}
- this.localsolroffset = this.query.itemsPerPage;
+ this.localsolroffset = this.query.offset + this.query.itemsPerPage;
// start a local RWI search concurrently
this.rwiProcess = null;
@@ -1576,31 +1578,68 @@ public final class SearchEvent {
// (happens if a search pages is accessed a second time)
final long finishTime = timeout == Long.MAX_VALUE ? Long.MAX_VALUE : System.currentTimeMillis() + timeout;
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "started, item = " + item + ", available = " + this.getResultCount(), 0, 0), false);
+
// wait until a local solr is finished, we must do that to be able to check if we need more
- if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join(100);} catch (final InterruptedException e) {}}
- if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
- if (item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
- // load remaining solr results now
+ if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {
+ try {
+ this.localsolrsearch.join(100);
+ } catch (final InterruptedException e) {
+ log.warn("Wait for local solr search was interrupted.");
+ }
+ }
+ if (item >= this.localsolroffset && this.local_solr_stored.get() == 0 && this.localsolrsearch.isAlive()) {
+ try {
+ this.localsolrsearch.join();
+ } catch (final InterruptedException e) {
+ log.warn("Wait for local solr search was interrupted.");
+ }
+ }
+ if (this.remote && item >= this.localsolroffset && this.local_solr_stored.get() >= item) {
+ /* Request mixing remote and local Solr results : load remaining local solr results now.
+ * For local only search, a new SearchEvent should be created, starting directly at the requested offset,
+ * thus allowing to handle last pages of large resultsets
+ */
int nextitems = item - this.localsolroffset + this.query.itemsPerPage; // example: suddenly switch to item 60, just 10 had been shown, 20 loaded.
if (this.localsolrsearch != null && this.localsolrsearch.isAlive()) {try {this.localsolrsearch.join();} catch (final InterruptedException e) {}}
- if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
- this.localsolrsearch = RemoteSearch.solrRemoteSearch(this, this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image), this.localsolroffset, nextitems, null /*this peer*/, 0, Switchboard.urlBlacklist);
- }
- this.localsolroffset += nextitems;
+ if (!Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.DEBUG_SEARCH_LOCAL_SOLR_OFF, false)) {
+ this.localsolrsearch = RemoteSearch.solrRemoteSearch(this,
+ this.query.solrQuery(this.query.contentdom, false, this.excludeintext_image),
+ this.localsolroffset, nextitems, null /* this peer */, 0, Switchboard.urlBlacklist);
+ }
+ this.localsolroffset += nextitems;
}
// now pull results as long as needed and as long as possible
- if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) try {Thread.sleep(100);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}
- while ( this.resultList.sizeAvailable() <= item &&
+ if (this.remote && item < 10 && this.resultList.sizeAvailable() <= item) {
+ try {
+ Thread.sleep(100);
+ } catch (final InterruptedException e) {
+ log.warn("Remote search results wait was interrupted.");
+ }
+ }
+
+ final int resultListIndex;
+ if (this.remote) {
+ resultListIndex = item;
+ } else {
+ resultListIndex = item - (this.localsolroffset - this.query.itemsPerPage);
+ }
+ while ( this.resultList.sizeAvailable() <= resultListIndex &&
(this.rwiQueueSize() > 0 || this.nodeStack.sizeQueue() > 0 ||
(!this.feedingIsFinished() && System.currentTimeMillis() < finishTime))) {
- if (!drainStacksToResult()) try {Thread.sleep(10);} catch (final InterruptedException e) {ConcurrentLog.logException(e);}
+ if (!drainStacksToResult()) {
+ try {
+ Thread.sleep(10);
+ } catch (final InterruptedException e) {
+ log.warn("Search results wait was interrupted.");
+ }
+ }
}
// check if we have a success
- if (this.resultList.sizeAvailable() > item) {
+ if (this.resultList.sizeAvailable() > resultListIndex) {
// we have the wanted result already in the result array .. return that
- final URIMetadataNode re = this.resultList.element(item).getElement();
+ final URIMetadataNode re = this.resultList.element(resultListIndex).getElement();
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(this.query.id(true), SearchEventType.ONERESULT, "fetched, item = " + item + ", available = " + this.getResultCount() + ": " + re.urlstring(), 0, 0), false);
/*
diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.java
index 0f8bb73fc..b8d9d4080 100644
--- a/source/net/yacy/search/query/SearchEventCache.java
+++ b/source/net/yacy/search/query/SearchEventCache.java
@@ -155,11 +155,24 @@ public class SearchEventCache {
event = null;
} else {
if (event != null) {
- //re-new the event time for this event, so it is not deleted next time too early
- event.resetEventTime();
- // replace the current result offset
- event.query.offset = query.offset;
- event.query.itemsPerPage = query.itemsPerPage;
+ if(query.isLocal()) {
+ /* Searching the local index only : we do not reuse the cached event each time the page size or offset changes.
+ * This allow to request last result pages of large result sets (larger than SearchEvent.max_results_node)
+ * without the need to retrieve all the beginning pages */
+ if(event.query.offset != query.offset || event.query.itemsPerPage != query.itemsPerPage) {
+ synchronized (lastEvents) {
+ lastEvents.remove(id);
+ }
+ cacheDelete++;
+ event = null;
+ }
+ } else {
+ //re-new the event time for this event, so it is not deleted next time too early
+ event.resetEventTime();
+ // replace the current result offset
+ event.query.offset = query.offset;
+ event.query.itemsPerPage = query.itemsPerPage;
+ }
}
}
if (event == null) {