Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Adjusted Solr select servlet output when using an external Solr only
luccioman26d8ad591c91
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.javanew file mode 100644index 000000000..0b8cfabe3--- /dev/null+++ b/source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.java@@ -0,0 +1,32 @@+// EmbeddedSolrResponseWriter.java+// ---------------------------+// Copyright 2018 by luccioman; https://github.com/luccioman+//+// This is a part of YaCy, a peer-to-peer based web search engine+//+// LICENSE+//+// This program is free software; you can redistribute it and/or modify+// it under the terms of the GNU General Public License as published by+// the Free Software Foundation; either version 2 of the License, or+// (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU General Public License for more details.+//+// You should have received a copy of the GNU General Public License+// along with this program; if not, write to the Free Software+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA++package net.yacy.cora.federate.solr.responsewriter;++import org.apache.solr.response.QueryResponseWriter;++/**+ * Interface used only to mark that a {@link QueryResponseWriter} implementation+ * class is only compatible with results from an embedded Solr server+ */+public interface EmbeddedSolrResponseWriter {+}diff --git a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.javaindex 9d05e00de..9358261d1 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java@@ -47,7 +47,7 @@ import org.json.simple.JSONArray;import net.yacy.cora.federate.solr.SolrType;import net.yacy.cora.util.JSONObject;-public class FlatJSONResponseWriter implements QueryResponseWriter {+public class FlatJSONResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {private static final char lb = '\n';diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.javaindex d5c7cc6da..7d4ac1842 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java@@ -59,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;* example: GET /gsa/searchresult?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1* for a xml reference, see https://developers.google.com/search-appliance/documentation/614/xml_reference*/-public class GSAResponseWriter implements QueryResponseWriter {+public class GSAResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {private static String YaCyVer = null;private static final char lb = '\n';diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.javaindex cdea36dad..6e95523cd 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java@@ -52,7 +52,7 @@ import org.apache.solr.search.SolrIndexSearcher;* text part and each sentence is shown as separate line. grep attributes can be used to* show leading and trainling lines.*/-public class GrepHTMLResponseWriter implements QueryResponseWriter {+public class GrepHTMLResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<String>();private static final Pattern dqp = Pattern.compile("\"");diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.javaindex 639685cbb..db5d08ebe 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java@@ -54,7 +54,7 @@ import net.yacy.cora.lod.vocabulary.DublinCore;import net.yacy.search.schema.CollectionSchema;import net.yacy.search.schema.WebgraphSchema;-public class HTMLResponseWriter implements QueryResponseWriter {+public class HTMLResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {public static final Pattern dqp = Pattern.compile("\"");diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.javaindex fd17e8f6a..83f0a1ab8 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java@@ -58,7 +58,7 @@ import org.apache.solr.search.DocIterator;import org.apache.solr.search.DocList;import org.apache.solr.search.SolrIndexSearcher;-public class OpensearchResponseWriter implements QueryResponseWriter {+public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {// define a list of simple YaCySchema -> RSS Token matchingsprivate static final Map<String, String> field2tag = new HashMap<String, String>();diff --git a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.javaindex 0ee342571..f40466ebf 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java@@ -20,7 +20,7 @@ import org.apache.solr.search.SolrIndexSearcher;/*** this writer is supposed to be used to generate iframes. It generates links for the /api/snapshot.jpg servlet.*/-public class SnapshotImagesReponseWriter implements QueryResponseWriter {+public class SnapshotImagesReponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<>();diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.javaindex 47dbb2e65..3380f537b 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java@@ -59,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;* example:* http://localhost:8090/solr/select?hl=false&wt=yjson&facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt&facet.field=collection_sxt&start=0&rows=10&query=www*/-public class YJsonResponseWriter implements QueryResponseWriter {+public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {// define a list of simple YaCySchema -> json Token matchingsprivate static final Map<String, String> field2tag = new HashMap<String, String>();diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.javaindex 7371816a2..fbfa84012 100644--- a/source/net/yacy/http/servlets/SolrSelectServlet.java+++ b/source/net/yacy/http/servlets/SolrSelectServlet.java@@ -40,6 +40,7 @@ import javax.servlet.http.HttpServletResponse;import net.yacy.cora.federate.solr.Ranking;import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;import net.yacy.cora.federate.solr.connector.SolrConnector;+import net.yacy.cora.federate.solr.responsewriter.EmbeddedSolrResponseWriter;import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter;import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter;@@ -60,17 +61,23 @@ import net.yacy.search.schema.CollectionSchema;import net.yacy.search.schema.WebgraphSchema;import org.apache.commons.lang.StringUtils;+import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.DisMaxParams;+import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.MultiMapSolrParams;import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.core.SolrCore;import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.request.SolrQueryRequestBase;import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.response.BinaryResponseWriter;+import org.apache.solr.response.CSVResponseWriter;import org.apache.solr.response.QueryResponseWriter;+import org.apache.solr.response.RawResponseWriter;import org.apache.solr.response.ResultContext;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.response.XSLTResponseWriter;@@ -287,16 +294,55 @@ public class SolrSelectServlet extends HttpServlet {out.flush();}} else {- // write a 'faked' response using a call to the backend- SolrDocumentList sdl = connector.getDocumentListByQuery(- mmsp.getMap().get(CommonParams.Q)[0],- mmsp.getMap().get(CommonParams.SORT) == null ? null : mmsp.getMap().get(CommonParams.SORT)[0],- Integer.parseInt(mmsp.getMap().get(CommonParams.START)[0]),- Integer.parseInt(mmsp.getMap().get(CommonParams.ROWS)[0]),- mmsp.getMap().get(CommonParams.FL));- OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream());- EnhancedXMLResponseWriter.write(osw, req, sdl);- osw.close();+ if (responseWriter instanceof EmbeddedSolrResponseWriter || responseWriter instanceof CSVResponseWriter+ || responseWriter instanceof XSLTResponseWriter || responseWriter instanceof RawResponseWriter) {+ /* These writers need a non null req.getSearcher(), req.getSchema() and/or req.getCore() */+ throw new ServletException("The writer " + responseWriter.getClass().getSimpleName() + " can only process responses from an embedded Solr server.");+ }++ QueryResponse queryRsp = connector.getResponseByParams(ModifiableSolrParams.of(mmsp));++ /* Create SolrQueryRequestBase and SolrQueryResponse instances as these types are requited by Solr standard writers.+ * WARNING : the SolrQueryRequestBase instance will return null for the getSearcher(), getCore() and getSchema() functions.+ * Be sure thath the responseWriter instance can handle this properly. */+ req = new SolrQueryRequestBase(null, mmsp) {};++ rsp = new SolrQueryResponse();+ NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();+ responseHeader.add("params", mmsp.toNamedList());+ rsp.add("responseHeader", responseHeader);+ rsp.setHttpCaching(false);+ rsp.getValues().addAll(queryRsp.getResponse());+++ // prepare response+ hresponse.setHeader("Cache-Control", "no-cache, no-store");++ final SolrDocumentList documentsList = queryRsp.getResults();+ long numFound = documentsList.getNumFound();+ AccessTracker.addToDump(querystring, numFound, new Date(), "sq");++ // write response header+ final String contentType = responseWriter.getContentType(req, rsp);+ if (null != contentType) {+ response.setContentType(contentType);+ }++ if (Method.HEAD == reqMethod) {+ return;+ }++ // write response body+ if (responseWriter instanceof EnhancedXMLResponseWriter) {+ out = new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);+ EnhancedXMLResponseWriter.write(out, req, documentsList);+ } else if(responseWriter instanceof BinaryResponseWriter) {+ ((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp);+ } else {+ out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8));+ responseWriter.write(out, req, rsp);+ out.flush();+ }}} catch (final Throwable ex) {sendError(hresponse, ex);@@ -305,7 +351,14 @@ public class SolrSelectServlet extends HttpServlet {req.close();}SolrRequestInfo.clearRequestInfo();- if (out != null) try {out.close();} catch (final IOException e1) {}+ if (out != null) {+ try {+ out.close();+ } catch (final IOException e1) {+ ConcurrentLog.info("SolrSelect", "Could not close output writer."+ + (e1.getMessage() != null ? "Cause : " + e1.getMessage() : ""));+ }+ }}}diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.javaindex 46b017626..d4ec3ba27 100644--- a/source/net/yacy/search/query/AccessTracker.java+++ b/source/net/yacy/search/query/AccessTracker.java@@ -151,13 +151,13 @@ public class AccessTracker {return 0;}- private static void addToDump(final QueryParams query, int resultCount) {+ private static void addToDump(final QueryParams query, long resultCount) {String queryString = query.getQueryGoal().getQueryString(false);if (queryString == null || queryString.isEmpty()) return;addToDump(queryString, resultCount, new Date(query.starttime), "qs");}- public static void addToDump(String querystring, int resultCount) {+ public static void addToDump(String querystring, long resultCount) {addToDump(querystring, resultCount, new Date(), "qs");}@@ -169,7 +169,7 @@ public class AccessTracker {* @param d start time* @param querySyntax used syntax (qs=normal querstring, sq=solr querystring,*/- public static void addToDump(String querystring, int resultcount, Date d, String querySyntax) {+ public static void addToDump(String querystring, long resultcount, Date d, String querySyntax) {//if (query.resultcount == 0) return;if (querystring == null || querystring.isEmpty()) return;final StringBuilder sb = new StringBuilder(40);