Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Handle Solr fields restrict and alias in YaCy html and exml writers
luccioman0d7625ecfb3c
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.javaindex f444688c3..7a687bef8 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java@@ -23,6 +23,7 @@ package net.yacy.cora.federate.solr.responsewriter;import java.io.IOException;import java.io.Writer;import java.util.Collection;+import java.util.Collections;import java.util.Date;import java.util.Iterator;import java.util.List;@@ -49,7 +50,9 @@ import org.apache.solr.schema.SchemaField;import org.apache.solr.schema.TextField;import org.apache.solr.search.DocIterator;import org.apache.solr.search.DocList;+import org.apache.solr.search.ReturnFields;import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.search.SolrReturnFields;public class EnhancedXMLResponseWriter implements QueryResponseWriter {@@ -83,14 +86,14 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {@SuppressWarnings("unchecked")SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) values.get("highlighting");writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader);- writeDocs(writer, request, response); // this.writeVal("response", response);+ writeDocs(writer, request, response, rsp.getReturnFields()); // this.writeVal("response", response);writeProps(writer, "highlighting", highlighting);writer.write(XML_STOP);}public static void write(final Writer writer, final SolrQueryRequest request, final SolrDocumentList sdl) throws IOException {writer.write(XML_START);- writeDocs(writer, request, sdl);+ writeDocs(writer, request, sdl, new SolrReturnFields(request));writer.write(XML_STOP);}@@ -113,7 +116,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {}}- private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response) throws IOException {+ private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response, final ReturnFields returnFields) throws IOException {boolean includeScore = false;final int sz = response.size();writer.write("<result");@@ -135,13 +138,13 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {for (int i = 0; i < sz; i++) {int id = iterator.nextDoc();Document doc = searcher.doc(id);- writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore);+ writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore, returnFields);}writer.write("</result>");writer.write(lb);}- private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs) throws IOException {+ private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs, final ReturnFields returnFields) throws IOException {boolean includeScore = false;final int sz = docs.size();writer.write("<result");@@ -159,18 +162,25 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {Iterator<SolrDocument> iterator = docs.iterator();for (int i = 0; i < sz; i++) {SolrDocument doc = iterator.next();- writeDoc(writer, doc);+ writeDoc(writer, doc, returnFields);}writer.write("</result>");writer.write(lb);}- private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final List<IndexableField> fields, final float score, final boolean includeScore) throws IOException {+ private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name,+ final List<IndexableField> fields, final float score, final boolean includeScore,+ final ReturnFields returnFields) throws IOException {startTagOpen(writer, "doc", name);if (includeScore) {writeTag(writer, "float", "score", Float.toString(score), false); // this is the special Solr "score" pseudo-field}++ /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter+ * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */+ final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()+ : returnFields.getFieldRenames();int sz = fields.size();int fidx1 = 0, fidx2 = 0;@@ -181,30 +191,35 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) {fidx2++;}- SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName);- if (sf == null) {- sf = new SchemaField(fieldName, new TextField());- }- FieldType type = sf.getType();- if (fidx1 + 1 == fidx2) {- if (sf.multiValued()) {- startTagOpen(writer, "arr", fieldName);- writer.write(lb);- String sv = value.stringValue();- writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);- writer.write("</arr>");- } else {- writeField(writer, type.getTypeName(), value.name(), value.stringValue()); //sf.write(this, f1.name(), f1);- }- } else {- startTagOpen(writer, "arr", fieldName);- writer.write(lb);- for (int i = fidx1; i < fidx2; i++) {- String sv = fields.get(i).stringValue();- writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));- }- writer.write("</arr>");- writer.write(lb);+ if(returnFields == null || returnFields.wantsField(fieldName)) {+ SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName);+ if (sf == null) {+ sf = new SchemaField(fieldName, new TextField());+ }++ final String renderedFieldName = fieldRenamings.getOrDefault(fieldName, fieldName);++ FieldType type = sf.getType();+ if (fidx1 + 1 == fidx2) {+ if (sf.multiValued()) {+ startTagOpen(writer, "arr", renderedFieldName);+ writer.write(lb);+ String sv = value.stringValue();+ writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);+ writer.write("</arr>");+ } else {+ writeField(writer, type.getTypeName(), renderedFieldName, value.stringValue()); //sf.write(this, f1.name(), f1);+ }+ } else {+ startTagOpen(writer, "arr", renderedFieldName);+ writer.write(lb);+ for (int i = fidx1; i < fidx2; i++) {+ String sv = fields.get(i).stringValue();+ writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));+ }+ writer.write("</arr>");+ writer.write(lb);+ }}fidx1 = fidx2;}@@ -233,27 +248,68 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {writer.write(lb);}- public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {+ /**+ * Append XML representation of the given Solr document to the writer+ *+ * @param writer+ * an open writer. Must not be null.+ * @param doc+ * the solr document to write. Must not be null.+ * @param returnFields+ * the eventual fields return configuration, allowing for example to+ * restrict the actually returned fields. May be null.+ * @throws IOException+ * when a write error occurred+ */+ private static final void writeDoc(final Writer writer, final SolrDocument doc, final ReturnFields returnFields) throws IOException {startTagOpen(writer, "doc", null);++ /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter+ * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */+ final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()+ : returnFields.getFieldRenames();+final Map<String, Object> fields = doc.getFieldValueMap();for (String key: fields.keySet()) {- if (key == null) continue;+ if (key == null) {+ continue;+ }+ if (returnFields != null && !returnFields.wantsField(key)) {+ continue;+ }Object value = doc.get(key);++ final String renderedFieldName = fieldRenamings.getOrDefault(key, key);+if (value == null) {} else if (value instanceof Collection<?>) {- startTagOpen(writer, "arr", key);+ startTagOpen(writer, "arr", renderedFieldName);writer.write(lb);for (Object o: ((Collection<?>) value)) {writeField(writer, null, o);}writer.write("</arr>"); writer.write(lb);} else {- writeField(writer, key, value);+ writeField(writer, renderedFieldName, value);}}writer.write("</doc>");writer.write(lb);}++ /**+ * Append XML representation of the given Solr document to the writer+ *+ * @param writer+ * an open writer. Must not be null.+ * @param doc+ * the solr document to write. Must not be null.+ * @throws IOException+ * when a write error occurred+ */+ public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {+ writeDoc(writer, doc, null);+ }private static void writeField(final Writer writer, final String typeName, final String name, final String value) throws IOException {if (typeName.equals(SolrType.text_general.printName()) ||diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.javaindex 679ca5cf5..639685cbb 100644--- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java+++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java@@ -24,17 +24,13 @@ import java.io.IOException;import java.io.Writer;import java.net.URLEncoder;import java.nio.charset.StandardCharsets;+import java.util.Collections;import java.util.Date;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.regex.Pattern;-import net.yacy.cora.federate.solr.SolrType;-import net.yacy.cora.lod.vocabulary.DublinCore;-import net.yacy.search.schema.CollectionSchema;-import net.yacy.search.schema.WebgraphSchema;-import org.apache.lucene.document.Document;import org.apache.lucene.index.IndexableField;import org.apache.solr.common.params.SolrParams;@@ -50,8 +46,14 @@ import org.apache.solr.schema.SchemaField;import org.apache.solr.schema.TextField;import org.apache.solr.search.DocIterator;import org.apache.solr.search.DocList;+import org.apache.solr.search.ReturnFields;import org.apache.solr.search.SolrIndexSearcher;+import net.yacy.cora.federate.solr.SolrType;+import net.yacy.cora.lod.vocabulary.DublinCore;+import net.yacy.search.schema.CollectionSchema;+import net.yacy.search.schema.WebgraphSchema;+public class HTMLResponseWriter implements QueryResponseWriter {public static final Pattern dqp = Pattern.compile("\"");@@ -165,7 +167,7 @@ public class HTMLResponseWriter implements QueryResponseWriter {int id = iterator.nextDoc();Document doc = searcher.doc(id);- LinkedHashMap<String, String> tdoc = translateDoc(schema, doc);+ LinkedHashMap<String, String> tdoc = translateDoc(schema, doc, rsp.getReturnFields());String title;@@ -187,19 +189,19 @@ public class HTMLResponseWriter implements QueryResponseWriter {writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</span></div>\n");- writeDoc(writer, tdoc, title, coreName);+ writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields());while (iterator.hasNext()) {id = iterator.nextDoc();doc = searcher.doc(id);- tdoc = translateDoc(schema, doc);+ tdoc = translateDoc(schema, doc, rsp.getReturnFields());if(CollectionSchema.CORE_NAME.equals(coreName)) {title = tdoc.get(CollectionSchema.title.getSolrFieldName());if (title == null) title = "";} else {title = "";}- writeDoc(writer, tdoc, title, coreName);+ writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields());}} else {writer.write("<title>No Document Found</title>\n</head><body>\n");@@ -209,7 +211,18 @@ public class HTMLResponseWriter implements QueryResponseWriter {writer.write("</body></html>\n");}- private static final void writeDoc(final Writer writer, final LinkedHashMap<String, String> tdoc, final String title, final String coreName) throws IOException {+ /**+ * Append an html representation of the document fields to the writer.+ * @param writer an open output writer. Must not be null.+ * @param tdoc the documents fields, mapped from field names to values. Must not be null.+ * @param title the document title. May be empty.+ * @param coreName the Solr core name.+ * @param returnFields the eventual fields return configuration, allowing for example to+ * rename fields with a pseudo in the result. May be null.+ * @throws IOException when a write error occurred.+ */+ private static final void writeDoc(final Writer writer, final LinkedHashMap<String, String> tdoc,+ final String title, final String coreName, final ReturnFields returnFields) throws IOException {writer.write("<form name=\"yacydoc" + title + "\" method=\"post\" action=\"#\" enctype=\"multipart/form-data\" accept-charset=\"UTF-8\">\n");writer.write("<fieldset>\n");@@ -225,9 +238,13 @@ public class HTMLResponseWriter implements QueryResponseWriter {}writer.write("<dl>\n");+ /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter+ * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */+ final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()+ : returnFields.getFieldRenames();for (Map.Entry<String, String> entry: tdoc.entrySet()) {writer.write("<dt>");- writer.write(entry.getKey());+ writer.write(fieldRenamings.getOrDefault(entry.getKey(), entry.getKey()));writer.write("</dt><dd>");if (entry.getKey().equals("sku")) {writer.write("<a href=\"" + entry.getValue() + "\">" + entry.getValue() + "</a>");@@ -241,7 +258,19 @@ public class HTMLResponseWriter implements QueryResponseWriter {writer.write("</form>\n");}- public static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc) {+ /**+ * Translate an indexed document in a map of field names to field values.+ *+ * @param schema+ * the schema of the indexed document. Must not be null.+ * @param doc+ * the indexed document. Must not be null.+ * @param returnFields+ * the eventual fields return configuration, allowing for example to+ * restrict the actually returned fields. May be null.+ * @return a map of field names to field values+ */+ private static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc, final ReturnFields returnFields) {List<IndexableField> fields = doc.getFields();int sz = fields.size();int fidx1 = 0, fidx2 = 0;@@ -253,29 +282,43 @@ public class HTMLResponseWriter implements QueryResponseWriter {while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) {fidx2++;}- SchemaField sf = schema.getFieldOrNull(fieldName);- if (sf == null) sf = new SchemaField(fieldName, new TextField());- FieldType type = sf.getType();+ if(returnFields == null || returnFields.wantsField(fieldName)) {+ SchemaField sf = schema.getFieldOrNull(fieldName);+ if (sf == null) {+ sf = new SchemaField(fieldName, new TextField());+ }+ FieldType type = sf.getType();- if (fidx1 + 1 == fidx2) {- if (sf.multiValued()) {- String sv = value.stringValue();- kv.put(fieldName, field2string(type, sv));- } else {- kv.put(fieldName, field2string(type, value.stringValue()));- }- } else {- int c = 0;- for (int i = fidx1; i < fidx2; i++) {- String sv = fields.get(i).stringValue();- kv.put(fieldName + "_" + c++, field2string(type, sv));- }+ if (fidx1 + 1 == fidx2) {+ if (sf.multiValued()) {+ String sv = value.stringValue();+ kv.put(fieldName, field2string(type, sv));+ } else {+ kv.put(fieldName, field2string(type, value.stringValue()));+ }+ } else {+ int c = 0;+ for (int i = fidx1; i < fidx2; i++) {+ String sv = fields.get(i).stringValue();+ kv.put(fieldName + "_" + c++, field2string(type, sv));+ }+ }}fidx1 = fidx2;}return kv;}++ /**+ * Translate an indexed document in a map of field names to field values.+ * @param schema the schema of the indexed document. Must not be null.+ * @param doc the indexed document. Must not be null.+ * @return a map of field names to field values+ */+ public static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc) {+ return translateDoc(schema, doc, null);+ }private static String field2string(final FieldType type, final String value) {String typeName = type.getTypeName();diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.javaindex 497eb1236..7371816a2 100644--- a/source/net/yacy/http/servlets/SolrSelectServlet.java+++ b/source/net/yacy/http/servlets/SolrSelectServlet.java@@ -99,8 +99,7 @@ public class SolrSelectServlet extends HttpServlet {RESPONSE_WRITER.putAll(SolrCore.DEFAULT_RESPONSE_WRITERS);XSLTResponseWriter xsltWriter = new XSLTResponseWriter();OpensearchResponseWriter opensearchResponseWriter = new OpensearchResponseWriter();- @SuppressWarnings("rawtypes")- NamedList initArgs = new NamedList();+ NamedList<?> initArgs = new NamedList<>();xsltWriter.init(initArgs);RESPONSE_WRITER.put("xslt", xsltWriter); // try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xslRESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter());