Skip to content

Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance

Handle Solr fields restrict and alias in YaCy html and exml writers

luccioman0d7625ecfb3c
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java
index f444688c3..7a687bef8 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/EnhancedXMLResponseWriter.java
@@ -23,6 +23,7 @@ package net.yacy.cora.federate.solr.responsewriter;
import java.io.IOException;
import java.io.Writer;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@@ -49,7 +50,9 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TextField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SolrReturnFields;
public class EnhancedXMLResponseWriter implements QueryResponseWriter {
@@ -83,14 +86,14 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
@SuppressWarnings("unchecked")
SimpleOrderedMap<Object> highlighting = (SimpleOrderedMap<Object>) values.get("highlighting");
writeProps(writer, "responseHeader", responseHeader); // this.writeVal("responseHeader", responseHeader);
- writeDocs(writer, request, response); // this.writeVal("response", response);
+ writeDocs(writer, request, response, rsp.getReturnFields()); // this.writeVal("response", response);
writeProps(writer, "highlighting", highlighting);
writer.write(XML_STOP);
}
public static void write(final Writer writer, final SolrQueryRequest request, final SolrDocumentList sdl) throws IOException {
writer.write(XML_START);
- writeDocs(writer, request, sdl);
+ writeDocs(writer, request, sdl, new SolrReturnFields(request));
writer.write(XML_STOP);
}
@@ -113,7 +116,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
}
}
- private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response) throws IOException {
+ private static final void writeDocs(final Writer writer, final SolrQueryRequest request, final DocList response, final ReturnFields returnFields) throws IOException {
boolean includeScore = false;
final int sz = response.size();
writer.write("<result");
@@ -135,13 +138,13 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
for (int i = 0; i < sz; i++) {
int id = iterator.nextDoc();
Document doc = searcher.doc(id);
- writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore);
+ writeDoc(writer, schema, null, doc.getFields(), (includeScore ? iterator.score() : 0.0f), includeScore, returnFields);
}
writer.write("</result>");
writer.write(lb);
}
- private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs) throws IOException {
+ private static final void writeDocs(final Writer writer, @SuppressWarnings("unused") final SolrQueryRequest request, final SolrDocumentList docs, final ReturnFields returnFields) throws IOException {
boolean includeScore = false;
final int sz = docs.size();
writer.write("<result");
@@ -159,18 +162,25 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
Iterator<SolrDocument> iterator = docs.iterator();
for (int i = 0; i < sz; i++) {
SolrDocument doc = iterator.next();
- writeDoc(writer, doc);
+ writeDoc(writer, doc, returnFields);
}
writer.write("</result>");
writer.write(lb);
}
- private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name, final List<IndexableField> fields, final float score, final boolean includeScore) throws IOException {
+ private static final void writeDoc(final Writer writer, final IndexSchema schema, final String name,
+ final List<IndexableField> fields, final float score, final boolean includeScore,
+ final ReturnFields returnFields) throws IOException {
startTagOpen(writer, "doc", name);
if (includeScore) {
writeTag(writer, "float", "score", Float.toString(score), false); // this is the special Solr "score" pseudo-field
}
+
+ /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter
+ * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */
+ final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()
+ : returnFields.getFieldRenames();
int sz = fields.size();
int fidx1 = 0, fidx2 = 0;
@@ -181,30 +191,35 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) {
fidx2++;
}
- SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName);
- if (sf == null) {
- sf = new SchemaField(fieldName, new TextField());
- }
- FieldType type = sf.getType();
- if (fidx1 + 1 == fidx2) {
- if (sf.multiValued()) {
- startTagOpen(writer, "arr", fieldName);
- writer.write(lb);
- String sv = value.stringValue();
- writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);
- writer.write("</arr>");
- } else {
- writeField(writer, type.getTypeName(), value.name(), value.stringValue()); //sf.write(this, f1.name(), f1);
- }
- } else {
- startTagOpen(writer, "arr", fieldName);
- writer.write(lb);
- for (int i = fidx1; i < fidx2; i++) {
- String sv = fields.get(i).stringValue();
- writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));
- }
- writer.write("</arr>");
- writer.write(lb);
+ if(returnFields == null || returnFields.wantsField(fieldName)) {
+ SchemaField sf = schema == null ? null : schema.getFieldOrNull(fieldName);
+ if (sf == null) {
+ sf = new SchemaField(fieldName, new TextField());
+ }
+
+ final String renderedFieldName = fieldRenamings.getOrDefault(fieldName, fieldName);
+
+ FieldType type = sf.getType();
+ if (fidx1 + 1 == fidx2) {
+ if (sf.multiValued()) {
+ startTagOpen(writer, "arr", renderedFieldName);
+ writer.write(lb);
+ String sv = value.stringValue();
+ writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, f1);
+ writer.write("</arr>");
+ } else {
+ writeField(writer, type.getTypeName(), renderedFieldName, value.stringValue()); //sf.write(this, f1.name(), f1);
+ }
+ } else {
+ startTagOpen(writer, "arr", renderedFieldName);
+ writer.write(lb);
+ for (int i = fidx1; i < fidx2; i++) {
+ String sv = fields.get(i).stringValue();
+ writeField(writer, type.getTypeName(), null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));
+ }
+ writer.write("</arr>");
+ writer.write(lb);
+ }
}
fidx1 = fidx2;
}
@@ -233,27 +248,68 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
writer.write(lb);
}
- public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {
+ /**
+ * Append XML representation of the given Solr document to the writer
+ *
+ * @param writer
+ * an open writer. Must not be null.
+ * @param doc
+ * the solr document to write. Must not be null.
+ * @param returnFields
+ * the eventual fields return configuration, allowing for example to
+ * restrict the actually returned fields. May be null.
+ * @throws IOException
+ * when a write error occurred
+ */
+ private static final void writeDoc(final Writer writer, final SolrDocument doc, final ReturnFields returnFields) throws IOException {
startTagOpen(writer, "doc", null);
+
+ /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter
+ * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */
+ final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()
+ : returnFields.getFieldRenames();
+
final Map<String, Object> fields = doc.getFieldValueMap();
for (String key: fields.keySet()) {
- if (key == null) continue;
+ if (key == null) {
+ continue;
+ }
+ if (returnFields != null && !returnFields.wantsField(key)) {
+ continue;
+ }
Object value = doc.get(key);
+
+ final String renderedFieldName = fieldRenamings.getOrDefault(key, key);
+
if (value == null) {
} else if (value instanceof Collection<?>) {
- startTagOpen(writer, "arr", key);
+ startTagOpen(writer, "arr", renderedFieldName);
writer.write(lb);
for (Object o: ((Collection<?>) value)) {
writeField(writer, null, o);
}
writer.write("</arr>"); writer.write(lb);
} else {
- writeField(writer, key, value);
+ writeField(writer, renderedFieldName, value);
}
}
writer.write("</doc>");
writer.write(lb);
}
+
+ /**
+ * Append XML representation of the given Solr document to the writer
+ *
+ * @param writer
+ * an open writer. Must not be null.
+ * @param doc
+ * the solr document to write. Must not be null.
+ * @throws IOException
+ * when a write error occurred
+ */
+ public static final void writeDoc(final Writer writer, final SolrDocument doc) throws IOException {
+ writeDoc(writer, doc, null);
+ }
private static void writeField(final Writer writer, final String typeName, final String name, final String value) throws IOException {
if (typeName.equals(SolrType.text_general.printName()) ||
diff --git a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
index 679ca5cf5..639685cbb 100644
--- a/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
+++ b/source/net/yacy/cora/federate/solr/responsewriter/HTMLResponseWriter.java
@@ -24,17 +24,13 @@ import java.io.IOException;
import java.io.Writer;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
-import net.yacy.cora.federate.solr.SolrType;
-import net.yacy.cora.lod.vocabulary.DublinCore;
-import net.yacy.search.schema.CollectionSchema;
-import net.yacy.search.schema.WebgraphSchema;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.params.SolrParams;
@@ -50,8 +46,14 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TextField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
+import net.yacy.cora.federate.solr.SolrType;
+import net.yacy.cora.lod.vocabulary.DublinCore;
+import net.yacy.search.schema.CollectionSchema;
+import net.yacy.search.schema.WebgraphSchema;
+
public class HTMLResponseWriter implements QueryResponseWriter {
public static final Pattern dqp = Pattern.compile("\"");
@@ -165,7 +167,7 @@ public class HTMLResponseWriter implements QueryResponseWriter {
int id = iterator.nextDoc();
Document doc = searcher.doc(id);
- LinkedHashMap<String, String> tdoc = translateDoc(schema, doc);
+ LinkedHashMap<String, String> tdoc = translateDoc(schema, doc, rsp.getReturnFields());
String title;
@@ -187,19 +189,19 @@ public class HTMLResponseWriter implements QueryResponseWriter {
writer.write("<div id=\"api\"><a href=\"" + xmlquery + "\"><img src=\"../env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>\n");
writer.write("<span>This search result can also be retrieved as XML. Click the API icon to see this page as XML.</span></div>\n");
- writeDoc(writer, tdoc, title, coreName);
+ writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields());
while (iterator.hasNext()) {
id = iterator.nextDoc();
doc = searcher.doc(id);
- tdoc = translateDoc(schema, doc);
+ tdoc = translateDoc(schema, doc, rsp.getReturnFields());
if(CollectionSchema.CORE_NAME.equals(coreName)) {
title = tdoc.get(CollectionSchema.title.getSolrFieldName());
if (title == null) title = "";
} else {
title = "";
}
- writeDoc(writer, tdoc, title, coreName);
+ writeDoc(writer, tdoc, title, coreName, rsp.getReturnFields());
}
} else {
writer.write("<title>No Document Found</title>\n</head><body>\n");
@@ -209,7 +211,18 @@ public class HTMLResponseWriter implements QueryResponseWriter {
writer.write("</body></html>\n");
}
- private static final void writeDoc(final Writer writer, final LinkedHashMap<String, String> tdoc, final String title, final String coreName) throws IOException {
+ /**
+ * Append an html representation of the document fields to the writer.
+ * @param writer an open output writer. Must not be null.
+ * @param tdoc the documents fields, mapped from field names to values. Must not be null.
+ * @param title the document title. May be empty.
+ * @param coreName the Solr core name.
+ * @param returnFields the eventual fields return configuration, allowing for example to
+ * rename fields with a pseudo in the result. May be null.
+ * @throws IOException when a write error occurred.
+ */
+ private static final void writeDoc(final Writer writer, final LinkedHashMap<String, String> tdoc,
+ final String title, final String coreName, final ReturnFields returnFields) throws IOException {
writer.write("<form name=\"yacydoc" + title + "\" method=\"post\" action=\"#\" enctype=\"multipart/form-data\" accept-charset=\"UTF-8\">\n");
writer.write("<fieldset>\n");
@@ -225,9 +238,13 @@ public class HTMLResponseWriter implements QueryResponseWriter {
}
writer.write("<dl>\n");
+ /* Fields may be renamed in the ouput result, using aliases in the 'fl' parameter
+ * (see https://lucene.apache.org/solr/guide/6_6/common-query-parameters.html#CommonQueryParameters-FieldNameAliases) */
+ final Map<String, String> fieldRenamings = returnFields == null ? Collections.emptyMap()
+ : returnFields.getFieldRenames();
for (Map.Entry<String, String> entry: tdoc.entrySet()) {
writer.write("<dt>");
- writer.write(entry.getKey());
+ writer.write(fieldRenamings.getOrDefault(entry.getKey(), entry.getKey()));
writer.write("</dt><dd>");
if (entry.getKey().equals("sku")) {
writer.write("<a href=\"" + entry.getValue() + "\">" + entry.getValue() + "</a>");
@@ -241,7 +258,19 @@ public class HTMLResponseWriter implements QueryResponseWriter {
writer.write("</form>\n");
}
- public static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc) {
+ /**
+ * Translate an indexed document in a map of field names to field values.
+ *
+ * @param schema
+ * the schema of the indexed document. Must not be null.
+ * @param doc
+ * the indexed document. Must not be null.
+ * @param returnFields
+ * the eventual fields return configuration, allowing for example to
+ * restrict the actually returned fields. May be null.
+ * @return a map of field names to field values
+ */
+ private static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc, final ReturnFields returnFields) {
List<IndexableField> fields = doc.getFields();
int sz = fields.size();
int fidx1 = 0, fidx2 = 0;
@@ -253,29 +282,43 @@ public class HTMLResponseWriter implements QueryResponseWriter {
while (fidx2 < sz && fieldName.equals(fields.get(fidx2).name())) {
fidx2++;
}
- SchemaField sf = schema.getFieldOrNull(fieldName);
- if (sf == null) sf = new SchemaField(fieldName, new TextField());
- FieldType type = sf.getType();
+ if(returnFields == null || returnFields.wantsField(fieldName)) {
+ SchemaField sf = schema.getFieldOrNull(fieldName);
+ if (sf == null) {
+ sf = new SchemaField(fieldName, new TextField());
+ }
+ FieldType type = sf.getType();
- if (fidx1 + 1 == fidx2) {
- if (sf.multiValued()) {
- String sv = value.stringValue();
- kv.put(fieldName, field2string(type, sv));
- } else {
- kv.put(fieldName, field2string(type, value.stringValue()));
- }
- } else {
- int c = 0;
- for (int i = fidx1; i < fidx2; i++) {
- String sv = fields.get(i).stringValue();
- kv.put(fieldName + "_" + c++, field2string(type, sv));
- }
+ if (fidx1 + 1 == fidx2) {
+ if (sf.multiValued()) {
+ String sv = value.stringValue();
+ kv.put(fieldName, field2string(type, sv));
+ } else {
+ kv.put(fieldName, field2string(type, value.stringValue()));
+ }
+ } else {
+ int c = 0;
+ for (int i = fidx1; i < fidx2; i++) {
+ String sv = fields.get(i).stringValue();
+ kv.put(fieldName + "_" + c++, field2string(type, sv));
+ }
+ }
}
fidx1 = fidx2;
}
return kv;
}
+
+ /**
+ * Translate an indexed document in a map of field names to field values.
+ * @param schema the schema of the indexed document. Must not be null.
+ * @param doc the indexed document. Must not be null.
+ * @return a map of field names to field values
+ */
+ public static final LinkedHashMap<String, String> translateDoc(final IndexSchema schema, final Document doc) {
+ return translateDoc(schema, doc, null);
+ }
private static String field2string(final FieldType type, final String value) {
String typeName = type.getTypeName();
diff --git a/source/net/yacy/http/servlets/SolrSelectServlet.java b/source/net/yacy/http/servlets/SolrSelectServlet.java
index 497eb1236..7371816a2 100644
--- a/source/net/yacy/http/servlets/SolrSelectServlet.java
+++ b/source/net/yacy/http/servlets/SolrSelectServlet.java
@@ -99,8 +99,7 @@ public class SolrSelectServlet extends HttpServlet {
RESPONSE_WRITER.putAll(SolrCore.DEFAULT_RESPONSE_WRITERS);
XSLTResponseWriter xsltWriter = new XSLTResponseWriter();
OpensearchResponseWriter opensearchResponseWriter = new OpensearchResponseWriter();
- @SuppressWarnings("rawtypes")
- NamedList initArgs = new NamedList();
+ NamedList<?> initArgs = new NamedList<>();
xsltWriter.init(initArgs);
RESPONSE_WRITER.put("xslt", xsltWriter); // try i.e. http://localhost:8090/solr/select?q=*:*&start=0&rows=10&wt=xslt&tr=json.xsl
RESPONSE_WRITER.put("exml", new EnhancedXMLResponseWriter());