Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Properly handle embedded Solr partial results
luccioman124cc24aa317
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.javaindex d7715cd36..3504048c2 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java@@ -39,7 +39,6 @@ import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.SolrInputField;import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.XML;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.QueryResponseWriter;@@ -77,18 +76,31 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter, SolrjResp@Overridepublic void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException {writer.write(XML_START);- NamedList<?> values = rsp.getValues();+ final NamedList<?> values = rsp.getValues();assert values.get("responseHeader") != null;assert values.get("response") != null;- SimpleOrderedMap<Object> responseHeader = (SimpleOrderedMap<Object>) rsp.getResponseHeader();- DocList response = ((ResultContext) values.get("response")).getDocList();- @SuppressWarnings("unchecked")- SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) values.get("highlighting");+ final NamedList<Object> responseHeader = rsp.getResponseHeader();+ final Object responseObj = rsp.getResponse();writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader);- writeDocs(writer, request, response, rsp.getReturnFields()); // this.writeVal("response", response);- writeProps(writer, "highlighting", highlighting);++ if(responseObj instanceof ResultContext) {+ /* Regular response object */+ writeDocs(writer, request, ((ResultContext) responseObj).getDocList(), rsp.getReturnFields());+ } else if(responseObj instanceof SolrDocumentList) {+ /*+ * The response object can be a SolrDocumentList when the response is partial,+ * for example when the allowed processing time has been exceeded+ */+ writeDocs(writer, (SolrDocumentList)responseObj, rsp.getReturnFields());+ } else {+ throw new IOException("Unable to process Solr response format");+ }+ final Object highlightingObj = values.get("highlighting");+ if(highlightingObj instanceof NamedList ) {+ writeProps(writer, "highlighting", (NamedList<?>)highlightingObj);+ }writer.write(XML_STOP);}diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.javaindex 362bc1385..aed07c29e 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java@@ -113,6 +113,20 @@ public class HTMLResponseWriter implements QueryResponseWriter, SolrjResponseWriwriter.write("*/");writer.write("</script>");}++ /**+ * Append a link to the related Solr API.+ * @param writer the output writer+ * @param paramsList the original request parameters+ * @param coreName the requested Solr core name+ * @throws IOException when a write error occurred+ */+ private void writeApiLink(final Writer writer, final NamedList<Object> paramsList, final String coreName) throws IOException {+ final String xmlquery = dqp.matcher("../solr/select?" + SolrParams.toSolrParams(paramsList).toString() + "&core=" + coreName).replaceAll("%22");++ writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");+ writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</span></div>\n");+ }@Overridepublic void write(final Writer writer, final SolrQueryRequest request, final SolrQueryResponse rsp) throws IOException {@@ -127,38 +141,55 @@ public class HTMLResponseWriter implements QueryResponseWriter, SolrjResponseWrifinal String coreName = request.getCore().getName();- String xmlquery = dqp.matcher("../solr/select?" + SolrParams.toSolrParams(paramsList).toString() + "&core=" + coreName).replaceAll("%22");-- DocList response = ((ResultContext) values.get("response")).getDocList();- final int sz = response.size();- if (sz > 0) {- SolrIndexSearcher searcher = request.getSearcher();- DocIterator iterator = response.iterator();- IndexSchema schema = request.getSchema();+ final Object responseObj = rsp.getResponse();+ final int sz;+ if(responseObj instanceof SolrDocumentList) {+ /*+ * The response object can be a SolrDocumentList when the response is partial,+ * for example when the allowed processing time has been exceeded+ */+ final SolrDocumentList docList = ((SolrDocumentList)responseObj);++ writeSolrDocumentList(writer, request, coreName, paramsList, docList);+ } else if(responseObj instanceof ResultContext){+ /* Regular response object */+ final DocList documents = ((ResultContext)responseObj).getDocList();++ sz = documents.size();++ if (sz > 0) {+ SolrIndexSearcher searcher = request.getSearcher();+ DocIterator iterator = documents.iterator();+ IndexSchema schema = request.getSchema();- int id = iterator.nextDoc();- Document doc = searcher.doc(id);- LinkedHashMap<String, String> tdoc = translateDoc(schema, doc, rsp.getReturnFields());+ int id = iterator.nextDoc();+ Document doc = searcher.doc(id);+ LinkedHashMap<String, String> tdoc = translateDoc(schema, doc, rsp.getReturnFields());- String title = doc.get(CollectionSchema.title.getSolrFieldName()); // title is multivalued, after translation fieldname could be in tdoc. "title_0" ..., so get it from doc- writeTitle(writer, coreName, sz, title);-- writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");- writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</span></div>\n");+ String title = doc.get(CollectionSchema.title.getSolrFieldName()); // title is multivalued, after translation fieldname could be in tdoc. "title_0" ..., so get it from doc+ writeTitle(writer, coreName, sz, title);++ writeApiLink(writer, paramsList, coreName);- writeDoc(writer, tdoc, coreName, rsp.getReturnFields());+ writeDoc(writer, tdoc, coreName, rsp.getReturnFields());- while (iterator.hasNext()) {- id = iterator.nextDoc();- doc = searcher.doc(id);- tdoc = translateDoc(schema, doc, rsp.getReturnFields());+ while (iterator.hasNext()) {+ id = iterator.nextDoc();+ doc = searcher.doc(id);+ tdoc = translateDoc(schema, doc, rsp.getReturnFields());- writeDoc(writer, tdoc, coreName, rsp.getReturnFields());+ writeDoc(writer, tdoc, coreName, rsp.getReturnFields());+ }+ } else {+ writer.write("<title>No Document Found</title>\n</head><body>\n");+ writer.write("<div class='alert alert-info'>No documents found</div>\n");}+} else {- writer.write("<title>No Document Found</title>\n</head><body>\n");- writer.write("<div class='alert alert-info'>No documents found</div>\n");+ writer.write("<title>Unable to process Solr response</title>\n</head><body>\n");+ writer.write("<div class='alert alert-info'>Unknown Solr response format</div>\n");}+writer.write("</body></html>\n");}@@ -171,14 +202,27 @@ public class HTMLResponseWriter implements QueryResponseWriter, SolrjResponseWrifinal NamedList<Object> paramsList = originalParams.toNamedList();paramsList.remove("wt");- final String xmlquery = dqp- .matcher("../solr/select?" + SolrParams.toSolrParams(paramsList).toString() + "&core=" + coreName)- .replaceAll("%22");-final SolrDocumentList docsList = rsp.getResults();- final int sz = docsList.size();++ writeSolrDocumentList(writer, request, coreName, paramsList, docsList);++ writer.write("</body></html>\n");+ }++ /**+ * Append to the writer HTML reprensentation of the given documents list.+ * @param writer the output writer+ * @param request the initial Solr request+ * @param coreName the requested Solr core+ * @param paramsList the original request parameters+ * @param docList the result Solr documents list+ * @throws IOException+ */+ private void writeSolrDocumentList(final Writer writer, final SolrQueryRequest request, final String coreName,+ final NamedList<Object> paramsList, final SolrDocumentList docList) throws IOException {+ final int sz = docList.size();if (sz > 0) {- final Iterator<SolrDocument> iterator = docsList.iterator();+ final Iterator<SolrDocument> iterator = docList.iterator();SolrDocument doc = iterator.next();final ReturnFields fieldsToReturn = request != null ? new SolrReturnFields(request) : new SolrReturnFields();@@ -187,10 +231,7 @@ public class HTMLResponseWriter implements QueryResponseWriter, SolrjResponseWrifinal String firstDocTitle = formatValue(titleValue);writeTitle(writer, coreName, sz, firstDocTitle);- writer.write("<div id=\"api\"><a href=\"" + xmlquery- + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");- writer.write(- "<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</span></div>\n");+ writeApiLink(writer, paramsList, coreName);writeDoc(writer, translateDoc(doc, fieldsToReturn), coreName, fieldsToReturn);@@ -203,8 +244,6 @@ public class HTMLResponseWriter implements QueryResponseWriter, SolrjResponseWriwriter.write("<title>No Document Found</title>\n</head><body>\n");writer.write("<div class='alert alert-info'>No documents found</div>\n");}-- writer.write("</body></html>\n");}/**diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.javaindex 95cfb9b73..7c336316a 100644--- a/source/net/yacy/http/servlets/SolrSelectServlet.java+++ b/source/net/yacy/http/servlets/SolrSelectServlet.java@@ -273,10 +273,19 @@ public class SolrSelectServlet extends HttpServlet {}- NamedList<?> values = rsp.getValues();- DocList r = ((ResultContext) values.get("response")).getDocList();- int numFound = r.matches();- AccessTracker.addToDump(querystring, numFound, new Date(), "sq");+ final Object responseObj = rsp.getResponse();+ if(responseObj instanceof ResultContext) {+ /* Regular response object */+ final DocList r = ((ResultContext) responseObj).getDocList();+ AccessTracker.addToDump(querystring, r.matches(), new Date(), "sq");+ } else if(responseObj instanceof SolrDocumentList){+ /*+ * The response object can be a SolrDocumentList when the response is partial,+ * for example when the allowed processing time has been exceeded+ */+ final SolrDocumentList r = (SolrDocumentList) responseObj;+ AccessTracker.addToDump(querystring, r.getNumFound(), new Date(), "sq");+ }// write response headerfinal String contentType = responseWriter.getContentType(req, rsp);