Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Adjusted Solr select servlet output when using an external Solr only

luccioman26d8ad591c91
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.java
new file mode 100644
index 000000000..0b8cfabe3
--- /dev/null
+++ b/source/net/yacy/cora/federate/solr/responsewriter/EmbeddedSolrResponseWriter.java
@@ -0,0 +1,32 @@
+// EmbeddedSolrResponseWriter.java
+// ---------------------------
+// Copyright 2018 by luccioman; https://github.com/luccioman
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.cora.federate.solr.responsewriter;
+
+import org.apache.solr.response.QueryResponseWriter;
+
+/**
+ * Interface used only to mark that a {@link QueryResponseWriter} implementation
+ * class is only compatible with results from an embedded Solr server
+ */
+public interface EmbeddedSolrResponseWriter {
+}
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java
index 9d05e00de..9358261d1 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/FlatJSONResponseWriter.java
@@ -47,7 +47,7 @@ import org.json.simple.JSONArray;
import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.util.JSONObject;
-public class FlatJSONResponseWriter implements QueryResponseWriter {
+public class FlatJSONResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
private static final char lb = '\n';
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java
index d5c7cc6da..7d4ac1842 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/GSAResponseWriter.java
@@ -59,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;
* example: GET /gsa/searchresult?q=chicken+teriyaki&output=xml&client=test&site=test&sort=date:D:S:d1
* for a xml reference, see https://developers.google.com/search-appliance/documentation/614/xml_reference
*/
-public class GSAResponseWriter implements QueryResponseWriter {
+public class GSAResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
private static String YaCyVer = null;
private static final char lb = '\n';
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java
index cdea36dad..6e95523cd 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/GrepHTMLResponseWriter.java
@@ -52,7 +52,7 @@ import org.apache.solr.search.SolrIndexSearcher;
* text part and each sentence is shown as separate line. grep attributes can be used to
* show leading and trainling lines.
*/
-public class GrepHTMLResponseWriter implements QueryResponseWriter {
+public class GrepHTMLResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<String>();
private static final Pattern dqp = Pattern.compile("\"");
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
index 639685cbb..db5d08ebe 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
@@ -54,7 +54,7 @@ import net.yacy.cora.lod.vocabulary.DublinCore;
import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
-public class HTMLResponseWriter implements QueryResponseWriter {
+public class HTMLResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
public static final Pattern dqp = Pattern.compile("\"");
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java
index fd17e8f6a..83f0a1ab8 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java
@@ -58,7 +58,7 @@ import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
-public class OpensearchResponseWriter implements QueryResponseWriter {
+public class OpensearchResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
// define a list of simple YaCySchema -> RSS Token matchings
private static final Map<String, String> field2tag = new HashMap<String, String>();
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java
index 0ee342571..f40466ebf 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/SnapshotImagesReponseWriter.java
@@ -20,7 +20,7 @@ import org.apache.solr.search.SolrIndexSearcher;
/**
* this writer is supposed to be used to generate iframes. It generates links for the /api/snapshot.jpg servlet.
*/
-public class SnapshotImagesReponseWriter implements QueryResponseWriter {
+public class SnapshotImagesReponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
private static final Set<String> DEFAULT_FIELD_LIST = new HashSet<>();
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
index 47dbb2e65..3380f537b 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java
@@ -59,7 +59,7 @@ import org.apache.solr.search.SolrIndexSearcher;
* example:
* http://localhost:8090/solr/select?hl=false&wt=yjson&facet=true&facet.mincount=1&facet.field=host_s&facet.field=url_file_ext_s&facet.field=url_protocol_s&facet.field=author_sxt&facet.field=collection_sxt&start=0&rows=10&query=www
*/
-public class YJsonResponseWriter implements QueryResponseWriter {
+public class YJsonResponseWriter implements QueryResponseWriter, EmbeddedSolrResponseWriter {
// define a list of simple YaCySchema -> json Token matchings
private static final Map<String, String> field2tag = new HashMap<String, String>();
diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java
index 7371816a2..fbfa84012 100644
--- a/source/net/yacy/http/servlets/SolrSelectServlet.java
+++ b/source/net/yacy/http/servlets/SolrSelectServlet.java
@@ -40,6 +40,7 @@ import javax.servlet.http.HttpServletResponse;
import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
+import net.yacy.cora.federate.solr.responsewriter.EmbeddedSolrResponseWriter;
import net.yacy.cora.federate.solr.responsewriter.EnhancedXMLResponseWriter;
import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter;
import net.yacy.cora.federate.solr.responsewriter.GrepHTMLResponseWriter;
@@ -60,17 +61,23 @@ import net.yacy.search.schema.CollectionSchema;
import net.yacy.search.schema.WebgraphSchema;
import org.apache.commons.lang.StringUtils;
+import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.DisMaxParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.BinaryResponseWriter;
+import org.apache.solr.response.CSVResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.response.XSLTResponseWriter;
@@ -287,16 +294,55 @@ public class SolrSelectServlet extends HttpServlet {
out.flush();
}
} else {
- // write a 'faked' response using a call to the backend
- SolrDocumentList sdl = connector.getDocumentListByQuery(
- mmsp.getMap().get(CommonParams.Q)[0],
- mmsp.getMap().get(CommonParams.SORT) == null ? null : mmsp.getMap().get(CommonParams.SORT)[0],
- Integer.parseInt(mmsp.getMap().get(CommonParams.START)[0]),
- Integer.parseInt(mmsp.getMap().get(CommonParams.ROWS)[0]),
- mmsp.getMap().get(CommonParams.FL));
- OutputStreamWriter osw = new OutputStreamWriter(response.getOutputStream());
- EnhancedXMLResponseWriter.write(osw, req, sdl);
- osw.close();
+ if (responseWriter instanceof EmbeddedSolrResponseWriter || responseWriter instanceof CSVResponseWriter
+ || responseWriter instanceof XSLTResponseWriter || responseWriter instanceof RawResponseWriter) {
+ /* These writers need a non null req.getSearcher(), req.getSchema() and/or req.getCore() */
+ throw new ServletException("The writer " + responseWriter.getClass().getSimpleName() + " can only process responses from an embedded Solr server.");
+ }
+
+ QueryResponse queryRsp = connector.getResponseByParams(ModifiableSolrParams.of(mmsp));
+
+ /* Create SolrQueryRequestBase and SolrQueryResponse instances as these types are requited by Solr standard writers.
+ * WARNING : the SolrQueryRequestBase instance will return null for the getSearcher(), getCore() and getSchema() functions.
+ * Be sure thath the responseWriter instance can handle this properly. */
+ req = new SolrQueryRequestBase(null, mmsp) {};
+
+ rsp = new SolrQueryResponse();
+ NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
+ responseHeader.add("params", mmsp.toNamedList());
+ rsp.add("responseHeader", responseHeader);
+ rsp.setHttpCaching(false);
+ rsp.getValues().addAll(queryRsp.getResponse());
+
+
+ // prepare response
+ hresponse.setHeader("Cache-Control", "no-cache, no-store");
+
+ final SolrDocumentList documentsList = queryRsp.getResults();
+ long numFound = documentsList.getNumFound();
+ AccessTracker.addToDump(querystring, numFound, new Date(), "sq");
+
+ // write response header
+ final String contentType = responseWriter.getContentType(req, rsp);
+ if (null != contentType) {
+ response.setContentType(contentType);
+ }
+
+ if (Method.HEAD == reqMethod) {
+ return;
+ }
+
+ // write response body
+ if (responseWriter instanceof EnhancedXMLResponseWriter) {
+ out = new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);
+ EnhancedXMLResponseWriter.write(out, req, documentsList);
+ } else if(responseWriter instanceof BinaryResponseWriter) {
+ ((BinaryResponseWriter) responseWriter).write(response.getOutputStream(), req, rsp);
+ } else {
+ out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8));
+ responseWriter.write(out, req, rsp);
+ out.flush();
+ }
}
} catch (final Throwable ex) {
sendError(hresponse, ex);
@@ -305,7 +351,14 @@ public class SolrSelectServlet extends HttpServlet {
req.close();
}
SolrRequestInfo.clearRequestInfo();
- if (out != null) try {out.close();} catch (final IOException e1) {}
+ if (out != null) {
+ try {
+ out.close();
+ } catch (final IOException e1) {
+ ConcurrentLog.info("SolrSelect", "Could not close output writer."
+ + (e1.getMessage() != null ? "Cause : " + e1.getMessage() : ""));
+ }
+ }
}
}
diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java
index 46b017626..d4ec3ba27 100644
--- a/source/net/yacy/search/query/AccessTracker.java
+++ b/source/net/yacy/search/query/AccessTracker.java
@@ -151,13 +151,13 @@ public class AccessTracker {
return 0;
}
- private static void addToDump(final QueryParams query, int resultCount) {
+ private static void addToDump(final QueryParams query, long resultCount) {
String queryString = query.getQueryGoal().getQueryString(false);
if (queryString == null || queryString.isEmpty()) return;
addToDump(queryString, resultCount, new Date(query.starttime), "qs");
}
- public static void addToDump(String querystring, int resultCount) {
+ public static void addToDump(String querystring, long resultCount) {
addToDump(querystring, resultCount, new Date(), "qs");
}
@@ -169,7 +169,7 @@ public class AccessTracker {
* @param d start time
* @param querySyntax used syntax (qs=normal querstring, sq=solr querystring,
*/
- public static void addToDump(String querystring, int resultcount, Date d, String querySyntax) {
+ public static void addToDump(String querystring, long resultcount, Date d, String querySyntax) {
//if (query.resultcount == 0) return;
if (querystring == null || querystring.isEmpty()) return;
final StringBuilder sb = new StringBuilder(40);