Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Made the dates navigator max elements number user configurable.
lucciomanef8aea7f8db9
diff --git a/defaults/yacy.init b/defaults/yacy.initindex a6365f18f..66c78683e 100644--- a/defaults/yacy.init+++ b/defaults/yacy.init@@ -885,6 +885,9 @@ search.navigation=location,hosts,authors,namespace,topics,filetype,protocol,lang# max number of items displayed in search navigatorssearch.navigation.maxcount=100+# max number of items displayed in the dates navigator+search.navigation.dates.maxcount=640+# search result verification and snippet fetch caching rules# each search result can be verified byloading the link from the web# this can be enhanced using a cache. In some cases it may be appropriatediff --git a/htroot/ConfigSearchPage_p.html b/htroot/ConfigSearchPage_p.htmlindex 7bd5b4eab..a95ab21ee 100644--- a/htroot/ConfigSearchPage_p.html+++ b/htroot/ConfigSearchPage_p.html@@ -179,13 +179,16 @@<input type="checkbox" name="search.navigation.date" value="true" #(search.navigation.date)#::checked="checked" #(/search.navigation.date)# /> Date Navigation+ <label title="Maximum days number in the histogram. Beware that a large value may trigger high CPU loads both on the server and on the browser with large result sets.">Maximum range (in days)+ <input type="number" name="search.navigation.dates.maxcount" value="#[search.navigation.dates.maxcount]#" min="6" max="2147483647"/>+ </label><link rel="stylesheet" href="env/morris.css"><script src="js/raphael.min.js"></script><script src="js/morris.js"></script><script type="text/javascript" src="js/accessibleHistogram.js" charset="UTF-8"></script><div id="graph" style="height:200px"></div><script>-var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=0&wt=json&facet=true&facet.field=dates_in_content_dts&facet.sort=index", function(data) {+var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=0&wt=json&facet=true&facet.field=dates_in_content_dts&facet.sort=index&facet.limit=#[search.navigation.dates.maxcount]#", function(data) {dates_in_content_dts = data.facet_counts.facet_fields.dates_in_content_dts;var parsed = [];for (var i = 0; i < dates_in_content_dts.length; i = i + 2) {diff --git a/htroot/ConfigSearchPage_p.java b/htroot/ConfigSearchPage_p.javaindex b86b5879b..42312c5b8 100644--- a/htroot/ConfigSearchPage_p.java+++ b/htroot/ConfigSearchPage_p.java@@ -104,11 +104,17 @@ public class ConfigSearchPage_p {if (nav.endsWith(",")) nav = nav.substring(0, nav.length() - 1);sb.setConfig("search.navigation", nav);// maxcount nav entries, default- int navmaxcnt = post.getInt("search.navigation.maxcount", QueryParams.FACETS_STANDARD_MAXCOUNT);+ int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);if (navmaxcnt > 5) {sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, navmaxcnt);- if (navmaxcnt != QueryParams.FACETS_STANDARD_MAXCOUNT) QueryParams.FACETS_STANDARD_MAXCOUNT = navmaxcnt;}++ // maxcount dates navigator entries+ int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT);+ if (datesNavMaxCnt > 5) {+ sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, datesNavMaxCnt);+ }}if (post.containsKey("add.nav")) { // button: add navigator plugin to ative list@@ -168,6 +174,12 @@ public class ConfigSearchPage_p {sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));sb.setConfig("search.result.show.hostbrowser", config.getProperty("search.result.show.hostbrowser","true"));sb.setConfig("search.result.show.snapshots", config.getProperty("search.result.show.snapshots","true"));+ sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,+ config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,+ String.valueOf(QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)));+ sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ String.valueOf(QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)));}}@@ -238,7 +250,11 @@ public class ConfigSearchPage_p {}prop.put("search.navigation.list", i);- prop.put("search.navigation.maxcount", sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT));+ prop.put(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, sb.getConfigInt(+ SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));++ prop.put(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, sb.getConfigInt(+ SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));prop.put("about.headline", sb.getConfig("about.headline", "About"));prop.put("about.body", sb.getConfig("about.body", ""));diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.javaindex 6c80e4346..ebe894c80 100644--- a/htroot/IndexControlRWIs_p.java+++ b/htroot/IndexControlRWIs_p.java@@ -679,7 +679,11 @@ public class IndexControlRWIs_p {sb.getRanking(),"",//userAgent0.0d, 0.0d, 0.0d,- new String[0]);+ new String[0]);+ query.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));+ query.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE);if (theSearch.rwiProcess != null && theSearch.rwiProcess.isAlive()) try {theSearch.rwiProcess.join();} catch (final InterruptedException e) {}if (theSearch.local_rwi_available.get() == 0) {diff --git a/htroot/yacy/search.java b/htroot/yacy/search.javaindex 46bfef3a4..80782b527 100644--- a/htroot/yacy/search.java+++ b/htroot/yacy/search.java@@ -255,6 +255,10 @@ public final class search {0.0d,new String[0]);+ theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));+ theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));Network.log.info("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()) + " - " + theQuery.itemsPerPage() + " links");final long timer = System.currentTimeMillis();diff --git a/htroot/yacysearch.java b/htroot/yacysearch.javaindex 1b8bc0c94..cac46a7b2 100644--- a/htroot/yacysearch.java+++ b/htroot/yacysearch.java@@ -653,6 +653,10 @@ public class yacysearch {header.get(HeaderFramework.USER_AGENT, ""),lat, lon, rad,sb.getConfigArray("search.navigation", ""));+ theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));+ theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));EventTracker.delete(EventTracker.EClass.SEARCH);EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theQuery.id(true),diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.javaindex 068a13be0..b1f5091b0 100644--- a/htroot/yacysearchtrailer.java+++ b/htroot/yacysearchtrailer.java@@ -110,7 +110,7 @@ public class yacysearchtrailer {navigatorIterator = languageNavigator.keys(false);int i = 0, pos = 0, neg = 0;String nav, rawNav;- while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {+ while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {name = navigatorIterator.next();count = languageNavigator.get(name);if (count == 0) break;@@ -202,7 +202,7 @@ public class yacysearchtrailer {if (oldProtocolModifier != null && oldProtocolModifier.length() > 0) {theSearch.query.modifier.remove("/" + oldProtocolModifier); theSearch.query.modifier.remove(oldProtocolModifier);}theSearch.query.modifier.protocol = "";theSearch.query.getQueryGoal().query_original = oldQuery.replaceAll(" /https", "").replaceAll(" /http", "").replaceAll(" /ftp", "").replaceAll(" /smb", "").replaceAll(" /file", "");- while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {+ while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {name = navigatorIterator.next().trim();count = theSearch.protocolNavigator.get(name);if (count == 0) break;@@ -253,7 +253,7 @@ public class yacysearchtrailer {if (fromconstraint == null) fromconstraint = new Date(System.currentTimeMillis() - AbstractFormatter.normalyearMillis);Date toconstraint = theSearch.getQuery().modifier.to == null ? null : DateDetection.parseLine(theSearch.getQuery().modifier.to, theSearch.getQuery().timezoneOffset);if (toconstraint == null) toconstraint = new Date(System.currentTimeMillis() + AbstractFormatter.normalyearMillis);- while (i < QueryParams.FACETS_DATE_MAXCOUNT && navigatorIterator.hasNext()) {+ while (i < theSearch.getQuery().getDateFacetMaxCount() && navigatorIterator.hasNext()) {name = navigatorIterator.next().trim();if (name.length() < 10) continue;count = theSearch.dateNavigator.get(name);@@ -266,7 +266,7 @@ public class yacysearchtrailer {if (fromconstraint != null && dd.before(fromconstraint)) continue;if (toconstraint != null && dd.after(toconstraint)) break;if (dx > 0) {- while (d - dx > AbstractFormatter.dayMillis) {+ while (d - dx > AbstractFormatter.dayMillis && i < theSearch.getQuery().getDateFacetMaxCount()) {dx += AbstractFormatter.dayMillis;String sn = new Date(dx).toInstant().toString().substring(0, 10);prop.put("nav-dates_element_" + i + "_on", 0);@@ -359,7 +359,7 @@ public class yacysearchtrailer {navigatorIterator = navi.keys(false);int i = 0, pos = 0, neg = 0;String nav, rawNav;- while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {+ while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {name = navigatorIterator.next();count = navi.get(name);if (count == 0) {diff --git a/source/net/yacy/cora/federate/FederateSearchManager.java b/source/net/yacy/cora/federate/FederateSearchManager.javaindex 132ecdd85..057ccdaa7 100644--- a/source/net/yacy/cora/federate/FederateSearchManager.java+++ b/source/net/yacy/cora/federate/FederateSearchManager.java@@ -223,6 +223,10 @@ public class FederateSearchManager {"",//userAgent0.0d, 0.0d, 0.0d,new String[0]);+ query.setStandardFacetsMaxCount(this.switchboard.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));+ query.setDateFacetMaxCount(this.switchboard.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));return query(query);}diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 3406071f2..80406b8b5 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -217,7 +217,6 @@ import net.yacy.search.index.Fulltext;import net.yacy.search.index.Segment;import net.yacy.search.index.Segment.ReferenceReportCache;import net.yacy.search.query.AccessTracker;-import net.yacy.search.query.QueryParams;import net.yacy.search.query.SearchEvent;import net.yacy.search.query.SearchEventCache;import net.yacy.search.ranking.RankingProfile;@@ -1235,9 +1234,6 @@ public final class Switchboard extends serverSwitch {this.trail = new LinkedBlockingQueue<String>();- // set configurable ui defaults- QueryParams.FACETS_STANDARD_MAXCOUNT = sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT); // max number of navigator/facet lines-this.log.config("Finished Switchboard Initialization");}diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.javaindex a5a1213ef..03b94c8b2 100644--- a/source/net/yacy/search/SwitchboardConstants.java+++ b/source/net/yacy/search/SwitchboardConstants.java@@ -553,7 +553,11 @@ public final class SwitchboardConstants {/** Default setting value controlling whether search results resorting by browser JavaScript is enabled */public static final boolean SEARCH_JS_RESORT_DEFAULT = false;- public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount"; // max lines displayed in standard search navigators/facets+ /** Key of the setting controlling the max lines displayed in standard search navigators/facets */+ public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount";++ /** Key of the setting controlling the max lines displayed in the dates navigator */+ public static final String SEARCH_NAVIGATION_DATES_MAXCOUNT = "search.navigation.dates.maxcount";/** Key of the setting controlling whether a noreferrer link type should be added to search result links */public static final String SEARCH_RESULT_NOREFERRER = "search.result.noreferrer";diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.javaindex 659f506b9..6ab48bb50 100644--- a/source/net/yacy/search/query/QueryParams.java+++ b/source/net/yacy/search/query/QueryParams.java@@ -80,8 +80,11 @@ import org.apache.solr.common.params.FacetParams;public final class QueryParams {- public static int FACETS_STANDARD_MAXCOUNT = 100; // max count of item lines in navigator- public static int FACETS_DATE_MAXCOUNT = 640;+ /** The default max count of item lines in navigator */+ public static final int FACETS_STANDARD_MAXCOUNT_DEFAULT = 100;++ /** The default maximum number of date elements in the date navigator */+ public static final int FACETS_DATE_MAXCOUNT_DEFAULT = 640;public enum Searchdom {LOCAL, CLUSTER, GLOBAL;@@ -148,6 +151,13 @@ public final class QueryParams {private SolrQuery cachedQuery;private CollectionConfiguration solrSchema;public final int timezoneOffset;++ /** The max count of item lines in navigator */+ private int standardFacetsMaxCount;++ /** The maximum number of date elements in the date navigator */+ private int dateFacetMaxCount;+public QueryParams(final QueryGoal queryGoal,@@ -269,6 +279,8 @@ public final class QueryParams {this.facetfields.add(CollectionSchema.VOCABULARY_PREFIX + context + CollectionSchema.VOCABULARY_TERMS_SUFFIX);}this.cachedQuery = null;+ this.standardFacetsMaxCount = FACETS_STANDARD_MAXCOUNT_DEFAULT;+ this.dateFacetMaxCount = FACETS_DATE_MAXCOUNT_DEFAULT;}private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid@@ -294,6 +306,34 @@ public final class QueryParams {public boolean isLocal() {return this.domType == Searchdom.LOCAL;}++ /**+ * @return the max count of item lines in standard navigators+ */+ public int getStandardFacetsMaxCount() {+ return this.standardFacetsMaxCount;+ }++ /**+ * @param standardFacetsMaxCount the max count of item lines in standard navigators+ */+ public void setStandardFacetsMaxCount(final int standardFacetsMaxCount) {+ this.standardFacetsMaxCount = standardFacetsMaxCount;+ }++ /**+ * @return the maximum number of date elements in the date navigator+ */+ public int getDateFacetMaxCount() {+ return this.dateFacetMaxCount;+ }++ /**+ * @param dateFacetMaxCount the maximum number of date elements in the date navigator+ */+ public void setDateFacetMaxCount(final int dateFacetMaxCount) {+ this.dateFacetMaxCount = dateFacetMaxCount;+ }public static HandleSet hashes2Set(final String query) {final HandleSet keyhashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);@@ -483,7 +523,7 @@ public final class QueryParams {if (getFacets && this.facetfields.size() > 0) {params.setFacet(true);params.setFacetMinCount(1);- params.setFacetLimit(FACETS_STANDARD_MAXCOUNT);+ params.setFacetLimit(this.standardFacetsMaxCount);params.setFacetSort(FacetParams.FACET_SORT_COUNT);params.setParam(FacetParams.FACET_METHOD, FacetParams.FACET_METHOD_enum); // fight the fieldcachefor (String field: this.facetfields) params.addFacetField("{!ex=" + field + "}" + field); // params.addFacetField("{!ex=" + field + "}" + field);@@ -495,7 +535,7 @@ public final class QueryParams {params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.end", end);params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.gap", "+1DAY");params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.sort", "index");- params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(FACETS_DATE_MAXCOUNT)); // the year constraint should cause that limitation already+ params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(this.dateFacetMaxCount)); // the year constraint should cause that limitation already}//for (String k: params.getParameterNames()) {ArrayList<String> al = new ArrayList<>(); for (String s: params.getParams(k)) al.add(s); System.out.println("Parameter: " + k + "=" + al.toString());}//http://localhost:8090/solr/collection1/select?q=*:*&rows=0&facet=true&facet.field=dates_in_content_dts&f.dates_in_content_dts.facet.limit=730&f.dates_in_content_dts.facet.sort=index