Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Added Solr filter queries for audio, video and application domains
luccioman66cb9c4ff9c7
diff --git a/source/net/yacy/search/query/QueryGoal.java b/source/net/yacy/search/query/QueryGoal.javaindex f20d8ffd1..b1ec4a5ec 100644--- a/source/net/yacy/search/query/QueryGoal.java+++ b/source/net/yacy/search/query/QueryGoal.java@@ -33,6 +33,8 @@ import java.util.Set;import java.util.SortedSet;import java.util.TreeSet;+import org.apache.http.HttpStatus;+import net.yacy.cora.document.WordCache;import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;import net.yacy.cora.order.NaturalOrder;@@ -345,7 +347,7 @@ public class QueryGoal {final ArrayList<String> fqs = new ArrayList<>();// add filter to prevent that results come from failed urls- fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");+ fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);if (noimages) {fqs.add("-" + CollectionSchema.content_type.getSolrFieldName() + ":(image/*)");fqs.add("-" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(jpg OR png OR gif)");@@ -379,13 +381,67 @@ public class QueryGoal {final ArrayList<String> fqs = new ArrayList<>();// add filter to prevent that results come from failed urls- fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200");+ fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);fqs.add(CollectionSchema.content_type.getSolrFieldName() + ":(image/*) OR " +CollectionSchema.images_urlstub_sxt.getSolrFieldName() + AbstractSolrConnector.CATCHALL_DTERM);return fqs;}+ /**+ * Generate Solr filter queries to receive valid video content results.+ *+ * This filters out documents with bad HTTP status and includes documents with MIME type matching the prefix video/* as well+ * docuemnts with links to video content.+ *+ * @return Solr filter queries for video content URLs+ */+ public List<String> collectionAudioFilterQuery() {+ final ArrayList<String> fqs = new ArrayList<>();++ // add filter to prevent that results come from failed urls+ fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);+ fqs.add(CollectionSchema.content_type.getSolrFieldName() + ":(audio/*) OR "+ + CollectionSchema.audiolinkscount_i.getSolrFieldName() + ":[1 TO *]");+ return fqs;+ }++ /**+ * Generate Solr filter queries to receive valid video content results.+ *+ * This filters out documents with bad HTTP status and includes documents with MIME type matching the prefix video/* as well+ * docuemnts with links to video content.+ *+ * @return Solr filter queries for video content URLs+ */+ public List<String> collectionVideoFilterQuery() {+ final ArrayList<String> fqs = new ArrayList<>();++ // add filter to prevent that results come from failed urls+ fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);+ fqs.add(CollectionSchema.content_type.getSolrFieldName() + ":(video/*) OR "+ + CollectionSchema.videolinkscount_i.getSolrFieldName() + ":[1 TO *]");+ return fqs;+ }++ /**+ * Generate Solr filter queries to receive valid application specific content results.+ *+ * This filters out documents with bad HTTP status and includes documents with MIME type matching the prefix application/* as well+ * docuemnts with links to application specific content.+ *+ * @return Solr filter queries for application specific content URLs+ */+ public List<String> collectionApplicationFilterQuery() {+ final ArrayList<String> fqs = new ArrayList<>();++ // add filter to prevent that results come from failed urls+ fqs.add(CollectionSchema.httpstatus_i.getSolrFieldName() + ":" + HttpStatus.SC_OK);+ fqs.add(CollectionSchema.content_type.getSolrFieldName() + ":(application/*) OR "+ + CollectionSchema.applinkscount_i.getSolrFieldName() + ":[1 TO *]");+ return fqs;+ }+public StringBuilder collectionImageQuery(final QueryModifier modifier) {final StringBuilder q = new StringBuilder(80);diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.javaindex 527631848..659f506b9 100644--- a/source/net/yacy/search/query/QueryParams.java+++ b/source/net/yacy/search/query/QueryParams.java@@ -111,7 +111,6 @@ public final class QueryParams {public static final Bitfield empty_constraint = new Bitfield(4, "AAAAAA");public static final Pattern catchall_pattern = Pattern.compile(".*");- private static final Pattern matchnothing_pattern = Pattern.compile("");private final QueryGoal queryGoal;public int itemsPerPage;@@ -370,11 +369,33 @@ public final class QueryParams {}public SolrQuery solrQuery(final ContentDomain cd, final boolean getFacets, final boolean excludeintext_image) {- if (cd == ContentDomain.IMAGE) return solrImageQuery(getFacets);- return solrTextQuery(getFacets, excludeintext_image);+ if (cd == ContentDomain.IMAGE) {+ return solrImageQuery(getFacets);+ }+ final List<String> filterQueries;+ switch (cd) {+ case AUDIO:+ filterQueries = this.queryGoal.collectionAudioFilterQuery();+ break;+ case VIDEO:+ filterQueries = this.queryGoal.collectionVideoFilterQuery();+ break;+ case APP:+ filterQueries = this.queryGoal.collectionApplicationFilterQuery();+ break;+ default:+ filterQueries = this.queryGoal.collectionTextFilterQuery(excludeintext_image);+ break;+ }+ return solrQuery(getFacets, filterQueries);}- private SolrQuery solrTextQuery(final boolean getFacets, final boolean excludeintext_image) {+ /**+ * @param getFacets when true, generate facets for fiels given in this.facetfields+ * @param filterQueries a mutable list of filter queries, initialized with filters related to content domain. Must not be null.+ * @return a Solr query instance ready to use+ */+ private SolrQuery solrQuery(final boolean getFacets, final List<String> filterQueries) {if (this.cachedQuery != null) {this.cachedQuery.setStart(this.offset);if (!getFacets) this.cachedQuery.setFacet(false);@@ -382,7 +403,7 @@ public final class QueryParams {}// construct query- final SolrQuery params = getBasicParams(getFacets, this.queryGoal.collectionTextFilterQuery(excludeintext_image));+ final SolrQuery params = getBasicParams(getFacets, filterQueries);int rankingProfile = this.ranking.coeff_date == RankingProfile.COEFF_MAX ? 1 : (this.modifier.sitehash != null || this.modifier.sitehost != null) ? 2 : 0;params.setQuery(this.queryGoal.collectionTextQuery().toString());Ranking actRanking = indexSegment.fulltext().getDefaultConfiguration().getRanking(rankingProfile); // for a by-date ranking select different ranking profile@@ -438,7 +459,7 @@ public final class QueryParams {return params;}- private SolrQuery getBasicParams(boolean getFacets, List<String> fqs) {+ private SolrQuery getBasicParams(final boolean getFacets, final List<String> fqs) {final SolrQuery params = new SolrQuery();params.setParam("defType", "edismax");params.setParam(DisMaxParams.QF, CollectionSchema.text_t.getSolrFieldName() + "^1.0");