Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Made the dates navigator max elements number user configurable.

lucciomanef8aea7f8db9
diff --git a/defaults/yacy.init b/defaults/yacy.init
index a6365f18f..66c78683e 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -885,6 +885,9 @@ search.navigation=location,hosts,authors,namespace,topics,filetype,protocol,lang
# max number of items displayed in search navigators
search.navigation.maxcount=100
+# max number of items displayed in the dates navigator
+search.navigation.dates.maxcount=640
+
# search result verification and snippet fetch caching rules
# each search result can be verified byloading the link from the web
# this can be enhanced using a cache. In some cases it may be appropriate
diff --git a/htroot/ConfigSearchPage_p.html b/htroot/ConfigSearchPage_p.html
index 7bd5b4eab..a95ab21ee 100644
--- a/htroot/ConfigSearchPage_p.html
+++ b/htroot/ConfigSearchPage_p.html
@@ -179,13 +179,16 @@
<input type="checkbox" name="search.navigation.date" value="true" #(search.navigation.date)#::checked="checked" #(/search.navigation.date)# /> Date Navigation
+ <label title="Maximum days number in the histogram. Beware that a large value may trigger high CPU loads both on the server and on the browser with large result sets.">Maximum range (in days)
+ <input type="number" name="search.navigation.dates.maxcount" value="#[search.navigation.dates.maxcount]#" min="6" max="2147483647"/>
+ </label>
<link rel="stylesheet" href="env/morris.css">
<script src="js/raphael.min.js"></script>
<script src="js/morris.js"></script>
<script type="text/javascript" src="js/accessibleHistogram.js" charset="UTF-8"></script>
<div id="graph" style="height:200px"></div>
<script>
-var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=0&wt=json&facet=true&facet.field=dates_in_content_dts&facet.sort=index", function(data) {
+var solr= $.getJSON("solr/collection1/select?q=*:*&defType=edismax&start=0&rows=0&wt=json&facet=true&facet.field=dates_in_content_dts&facet.sort=index&facet.limit=#[search.navigation.dates.maxcount]#", function(data) {
dates_in_content_dts = data.facet_counts.facet_fields.dates_in_content_dts;
var parsed = [];
for (var i = 0; i < dates_in_content_dts.length; i = i + 2) {
diff --git a/htroot/ConfigSearchPage_p.java b/htroot/ConfigSearchPage_p.java
index b86b5879b..42312c5b8 100644
--- a/htroot/ConfigSearchPage_p.java
+++ b/htroot/ConfigSearchPage_p.java
@@ -104,11 +104,17 @@ public class ConfigSearchPage_p {
if (nav.endsWith(",")) nav = nav.substring(0, nav.length() - 1);
sb.setConfig("search.navigation", nav);
// maxcount nav entries, default
- int navmaxcnt = post.getInt("search.navigation.maxcount", QueryParams.FACETS_STANDARD_MAXCOUNT);
+ int navmaxcnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT);
if (navmaxcnt > 5) {
sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, navmaxcnt);
- if (navmaxcnt != QueryParams.FACETS_STANDARD_MAXCOUNT) QueryParams.FACETS_STANDARD_MAXCOUNT = navmaxcnt;
}
+
+ // maxcount dates navigator entries
+ int datesNavMaxCnt = post.getInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT);
+ if (datesNavMaxCnt > 5) {
+ sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, datesNavMaxCnt);
+ }
}
if (post.containsKey("add.nav")) { // button: add navigator plugin to ative list
@@ -168,6 +174,12 @@ public class ConfigSearchPage_p {
sb.setConfig("search.result.show.proxy", config.getProperty("search.result.show.proxy","false"));
sb.setConfig("search.result.show.hostbrowser", config.getProperty("search.result.show.hostbrowser","true"));
sb.setConfig("search.result.show.snapshots", config.getProperty("search.result.show.snapshots","true"));
+ sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
+ config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
+ String.valueOf(QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT)));
+ sb.setConfig(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ config.getProperty(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ String.valueOf(QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)));
}
}
@@ -238,7 +250,11 @@ public class ConfigSearchPage_p {
}
prop.put("search.navigation.list", i);
- prop.put("search.navigation.maxcount", sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT));
+ prop.put(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, sb.getConfigInt(
+ SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
+
+ prop.put(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, sb.getConfigInt(
+ SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
prop.put("about.headline", sb.getConfig("about.headline", "About"));
prop.put("about.body", sb.getConfig("about.body", ""));
diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java
index 6c80e4346..ebe894c80 100644
--- a/htroot/IndexControlRWIs_p.java
+++ b/htroot/IndexControlRWIs_p.java
@@ -679,7 +679,11 @@ public class IndexControlRWIs_p {
sb.getRanking(),
"",//userAgent
0.0d, 0.0d, 0.0d,
- new String[0]);
+ new String[0]);
+ query.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
+ query.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
final SearchEvent theSearch = SearchEventCache.getEvent(query, sb.peers, sb.tables, null, false, sb.loader, Integer.MAX_VALUE, Long.MAX_VALUE);
if (theSearch.rwiProcess != null && theSearch.rwiProcess.isAlive()) try {theSearch.rwiProcess.join();} catch (final InterruptedException e) {}
if (theSearch.local_rwi_available.get() == 0) {
diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java
index 46bfef3a4..80782b527 100644
--- a/htroot/yacy/search.java
+++ b/htroot/yacy/search.java
@@ -255,6 +255,10 @@ public final class search {
0.0d,
new String[0]
);
+ theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
+ theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
Network.log.info("INIT HASH SEARCH (abstracts only): " + QueryParams.anonymizedQueryHashes(theQuery.getQueryGoal().getIncludeHashes()) + " - " + theQuery.itemsPerPage() + " links");
final long timer = System.currentTimeMillis();
diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java
index 1b8bc0c94..cac46a7b2 100644
--- a/htroot/yacysearch.java
+++ b/htroot/yacysearch.java
@@ -653,6 +653,10 @@ public class yacysearch {
header.get(HeaderFramework.USER_AGENT, ""),
lat, lon, rad,
sb.getConfigArray("search.navigation", ""));
+ theQuery.setStandardFacetsMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
+ theQuery.setDateFacetMaxCount(sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
EventTracker.delete(EventTracker.EClass.SEARCH);
EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(
theQuery.id(true),
diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java
index 068a13be0..b1f5091b0 100644
--- a/htroot/yacysearchtrailer.java
+++ b/htroot/yacysearchtrailer.java
@@ -110,7 +110,7 @@ public class yacysearchtrailer {
navigatorIterator = languageNavigator.keys(false);
int i = 0, pos = 0, neg = 0;
String nav, rawNav;
- while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
+ while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = languageNavigator.get(name);
if (count == 0) break;
@@ -202,7 +202,7 @@ public class yacysearchtrailer {
if (oldProtocolModifier != null && oldProtocolModifier.length() > 0) {theSearch.query.modifier.remove("/" + oldProtocolModifier); theSearch.query.modifier.remove(oldProtocolModifier);}
theSearch.query.modifier.protocol = "";
theSearch.query.getQueryGoal().query_original = oldQuery.replaceAll(" /https", "").replaceAll(" /http", "").replaceAll(" /ftp", "").replaceAll(" /smb", "").replaceAll(" /file", "");
- while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
+ while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next().trim();
count = theSearch.protocolNavigator.get(name);
if (count == 0) break;
@@ -253,7 +253,7 @@ public class yacysearchtrailer {
if (fromconstraint == null) fromconstraint = new Date(System.currentTimeMillis() - AbstractFormatter.normalyearMillis);
Date toconstraint = theSearch.getQuery().modifier.to == null ? null : DateDetection.parseLine(theSearch.getQuery().modifier.to, theSearch.getQuery().timezoneOffset);
if (toconstraint == null) toconstraint = new Date(System.currentTimeMillis() + AbstractFormatter.normalyearMillis);
- while (i < QueryParams.FACETS_DATE_MAXCOUNT && navigatorIterator.hasNext()) {
+ while (i < theSearch.getQuery().getDateFacetMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next().trim();
if (name.length() < 10) continue;
count = theSearch.dateNavigator.get(name);
@@ -266,7 +266,7 @@ public class yacysearchtrailer {
if (fromconstraint != null && dd.before(fromconstraint)) continue;
if (toconstraint != null && dd.after(toconstraint)) break;
if (dx > 0) {
- while (d - dx > AbstractFormatter.dayMillis) {
+ while (d - dx > AbstractFormatter.dayMillis && i < theSearch.getQuery().getDateFacetMaxCount()) {
dx += AbstractFormatter.dayMillis;
String sn = new Date(dx).toInstant().toString().substring(0, 10);
prop.put("nav-dates_element_" + i + "_on", 0);
@@ -359,7 +359,7 @@ public class yacysearchtrailer {
navigatorIterator = navi.keys(false);
int i = 0, pos = 0, neg = 0;
String nav, rawNav;
- while (i < QueryParams.FACETS_STANDARD_MAXCOUNT && navigatorIterator.hasNext()) {
+ while (i < theSearch.getQuery().getStandardFacetsMaxCount() && navigatorIterator.hasNext()) {
name = navigatorIterator.next();
count = navi.get(name);
if (count == 0) {
diff --git a/source/net/yacy/cora/federate/FederateSearchManager.java b/source/net/yacy/cora/federate/FederateSearchManager.java
index 132ecdd85..057ccdaa7 100644
--- a/source/net/yacy/cora/federate/FederateSearchManager.java
+++ b/source/net/yacy/cora/federate/FederateSearchManager.java
@@ -223,6 +223,10 @@ public class FederateSearchManager {
"",//userAgent
0.0d, 0.0d, 0.0d,
new String[0]);
+ query.setStandardFacetsMaxCount(this.switchboard.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT,
+ QueryParams.FACETS_STANDARD_MAXCOUNT_DEFAULT));
+ query.setDateFacetMaxCount(this.switchboard.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
+ QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
return query(query);
}
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index 3406071f2..80406b8b5 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -217,7 +217,6 @@ import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Segment;
import net.yacy.search.index.Segment.ReferenceReportCache;
import net.yacy.search.query.AccessTracker;
-import net.yacy.search.query.QueryParams;
import net.yacy.search.query.SearchEvent;
import net.yacy.search.query.SearchEventCache;
import net.yacy.search.ranking.RankingProfile;
@@ -1235,9 +1234,6 @@ public final class Switchboard extends serverSwitch {
this.trail = new LinkedBlockingQueue<String>();
- // set configurable ui defaults
- QueryParams.FACETS_STANDARD_MAXCOUNT = sb.getConfigInt(SwitchboardConstants.SEARCH_NAVIGATION_MAXCOUNT, QueryParams.FACETS_STANDARD_MAXCOUNT); // max number of navigator/facet lines
-
this.log.config("Finished Switchboard Initialization");
}
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index a5a1213ef..03b94c8b2 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -553,7 +553,11 @@ public final class SwitchboardConstants {
/** Default setting value controlling whether search results resorting by browser JavaScript is enabled */
public static final boolean SEARCH_JS_RESORT_DEFAULT = false;
- public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount"; // max lines displayed in standard search navigators/facets
+ /** Key of the setting controlling the max lines displayed in standard search navigators/facets */
+ public static final String SEARCH_NAVIGATION_MAXCOUNT = "search.navigation.maxcount";
+
+ /** Key of the setting controlling the max lines displayed in the dates navigator */
+ public static final String SEARCH_NAVIGATION_DATES_MAXCOUNT = "search.navigation.dates.maxcount";
/** Key of the setting controlling whether a noreferrer link type should be added to search result links */
public static final String SEARCH_RESULT_NOREFERRER = "search.result.noreferrer";
diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java
index 659f506b9..6ab48bb50 100644
--- a/source/net/yacy/search/query/QueryParams.java
+++ b/source/net/yacy/search/query/QueryParams.java
@@ -80,8 +80,11 @@ import org.apache.solr.common.params.FacetParams;
public final class QueryParams {
- public static int FACETS_STANDARD_MAXCOUNT = 100; // max count of item lines in navigator
- public static int FACETS_DATE_MAXCOUNT = 640;
+ /** The default max count of item lines in navigator */
+ public static final int FACETS_STANDARD_MAXCOUNT_DEFAULT = 100;
+
+ /** The default maximum number of date elements in the date navigator */
+ public static final int FACETS_DATE_MAXCOUNT_DEFAULT = 640;
public enum Searchdom {
LOCAL, CLUSTER, GLOBAL;
@@ -148,6 +151,13 @@ public final class QueryParams {
private SolrQuery cachedQuery;
private CollectionConfiguration solrSchema;
public final int timezoneOffset;
+
+ /** The max count of item lines in navigator */
+ private int standardFacetsMaxCount;
+
+ /** The maximum number of date elements in the date navigator */
+ private int dateFacetMaxCount;
+
public QueryParams(
final QueryGoal queryGoal,
@@ -269,6 +279,8 @@ public final class QueryParams {
this.facetfields.add(CollectionSchema.VOCABULARY_PREFIX + context + CollectionSchema.VOCABULARY_TERMS_SUFFIX);
}
this.cachedQuery = null;
+ this.standardFacetsMaxCount = FACETS_STANDARD_MAXCOUNT_DEFAULT;
+ this.dateFacetMaxCount = FACETS_DATE_MAXCOUNT_DEFAULT;
}
private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid
@@ -294,6 +306,34 @@ public final class QueryParams {
public boolean isLocal() {
return this.domType == Searchdom.LOCAL;
}
+
+ /**
+ * @return the max count of item lines in standard navigators
+ */
+ public int getStandardFacetsMaxCount() {
+ return this.standardFacetsMaxCount;
+ }
+
+ /**
+ * @param standardFacetsMaxCount the max count of item lines in standard navigators
+ */
+ public void setStandardFacetsMaxCount(final int standardFacetsMaxCount) {
+ this.standardFacetsMaxCount = standardFacetsMaxCount;
+ }
+
+ /**
+ * @return the maximum number of date elements in the date navigator
+ */
+ public int getDateFacetMaxCount() {
+ return this.dateFacetMaxCount;
+ }
+
+ /**
+ * @param dateFacetMaxCount the maximum number of date elements in the date navigator
+ */
+ public void setDateFacetMaxCount(final int dateFacetMaxCount) {
+ this.dateFacetMaxCount = dateFacetMaxCount;
+ }
public static HandleSet hashes2Set(final String query) {
final HandleSet keyhashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
@@ -483,7 +523,7 @@ public final class QueryParams {
if (getFacets && this.facetfields.size() > 0) {
params.setFacet(true);
params.setFacetMinCount(1);
- params.setFacetLimit(FACETS_STANDARD_MAXCOUNT);
+ params.setFacetLimit(this.standardFacetsMaxCount);
params.setFacetSort(FacetParams.FACET_SORT_COUNT);
params.setParam(FacetParams.FACET_METHOD, FacetParams.FACET_METHOD_enum); // fight the fieldcache
for (String field: this.facetfields) params.addFacetField("{!ex=" + field + "}" + field); // params.addFacetField("{!ex=" + field + "}" + field);
@@ -495,7 +535,7 @@ public final class QueryParams {
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.end", end);
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.gap", "+1DAY");
params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.sort", "index");
- params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(FACETS_DATE_MAXCOUNT)); // the year constraint should cause that limitation already
+ params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(this.dateFacetMaxCount)); // the year constraint should cause that limitation already
}
//for (String k: params.getParameterNames()) {ArrayList<String> al = new ArrayList<>(); for (String s: params.getParams(k)) al.add(s); System.out.println("Parameter: " + k + "=" + al.toString());}
//http://localhost:8090/solr/collection1/select?q=*:*&rows=0&facet=true&facet.field=dates_in_content_dts&f.dates_in_content_dts.facet.limit=730&f.dates_in_content_dts.facet.sort=index