Apache Solr open-source search software
SOLR-9957: /export supports document transformers in fl (#5113)
David Smileybd2b2d5a0258
diff --git a/changelog/unreleased/SOLR-9957-exportTransformers.yml b/changelog/unreleased/SOLR-9957-exportTransformers.ymlnew file mode 100644index 00000000000..aa37e8f4183--- /dev/null+++ b/changelog/unreleased/SOLR-9957-exportTransformers.yml@@ -0,0 +1,10 @@+title: >+ The /export handler now supports document transformers (e.g. `[docid]`) and field aliases in `fl`,+ and their output is visible to its local streaming expression (`expr`). An aliased field such as+ `fl=x:myfield` is now returned as `x`; previously it was silently returned as `myfield`.+type: added+authors:+ - name: David Smiley+links:+ - name: SOLR-9957+ url: https://issues.apache.org/jira/browse/SOLR-9957diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.javaindex 7b82ad28b14..7872a61be1f 100644--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java@@ -26,6 +26,7 @@ import java.io.PrintWriter;import java.lang.invoke.MethodHandles;import java.nio.charset.StandardCharsets;import java.util.ArrayList;+import java.util.HashSet;import java.util.LinkedHashMap;import java.util.LinkedHashSet;import java.util.List;@@ -36,6 +37,7 @@ import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.Query;import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.lucene.util.BitSetIterator;@@ -52,10 +54,12 @@ import org.apache.solr.common.IteratorWriter;import org.apache.solr.common.MapWriter;import org.apache.solr.common.MapWriter.EntryWriter;import org.apache.solr.common.PushWriter;+import org.apache.solr.common.SolrDocument;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.params.StreamParams;import org.apache.solr.common.util.JavaBinCodec;+import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.core.SolrCore;import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.request.SolrQueryRequest;@@ -64,7 +68,9 @@ import org.apache.solr.response.JSONResponseWriter;import org.apache.solr.response.JacksonJsonWriter;import org.apache.solr.response.JavaBinResponseWriter;import org.apache.solr.response.QueryResponseWriter;+import org.apache.solr.response.ResultContext;import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.response.transform.DocTransformer;import org.apache.solr.schema.BoolField;import org.apache.solr.schema.DateValueFieldType;import org.apache.solr.schema.DoubleValueFieldType;@@ -76,7 +82,10 @@ import org.apache.solr.schema.LongValueFieldType;import org.apache.solr.schema.SchemaField;import org.apache.solr.schema.SortableTextField;import org.apache.solr.schema.StrField;+import org.apache.solr.search.DocIterationInfo;+import org.apache.solr.search.DocList;import org.apache.solr.search.DocValuesIteratorCache;+import org.apache.solr.search.ReturnFields;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.search.SolrReturnFields;import org.apache.solr.search.SortSpec;@@ -124,7 +133,9 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {private int priorityQueueSize;StreamExpression streamExpression;StreamContext streamContext;+ SolrReturnFields returnFields;List<FieldWriter> fieldWriters;+ DocTransformer transformer;int totalHits = 0;FixedBitSet[] sets = null;PushWriter writer;@@ -268,35 +279,38 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {}}SolrParams params = req.getParams();- String fl = params.get("fl");- String[] fields = null;-- if (fl == null) {+ if (params.get(CommonParams.FL) == null) {writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))),writer,true);return;- } else {- fields = fl.split(",");-- for (int i = 0; i < fields.length; i++) {+ }- fields[i] = fields[i].trim();+ try {+ returnFields = new SolrReturnFields(req);+ } catch (Exception e) {+ writeException(e, writer, true);+ return;+ }- if (fields[i].equals("score")) {- writeException(- (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))),- writer,- true);- return;- }- }+ if (returnFields.wantsScore()) {+ writeException(+ (new IOException(new SyntaxError("Scoring is not currently supported with xsort."))),+ writer,+ true);+ return;}try {- fieldWriters = getFieldWriters(fields, req);+ fieldWriters = getFieldWriters(returnFields, req);+ transformer = returnFields.getTransformer();+ if (transformer != null) {+ transformer.setContext(+ new ExportResultContext(+ returnFields, searcher, info.getResponseBuilder().getQuery(), req));+ }} catch (Exception e) {writeException(e, writer, true);return;@@ -483,33 +497,73 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {throws IOException {int ord = sortDoc.ord;LeafReaderContext context = leaves.get(ord);+ if (transformer == null) {+ for (FieldWriter fieldWriter : writers) {+ fieldWriter.write(sortDoc, context, ew);+ }+ return;+ }++ SolrDocument doc = new SolrDocument();+ EntryWriter docWriter =+ new EntryWriter() {+ @Override+ public EntryWriter put(CharSequence k, Object v) {+ doc.setField(k.toString(), materialize(v));+ return this;+ }+ };for (FieldWriter fieldWriter : writers) {- fieldWriter.write(sortDoc, context, ew);+ fieldWriter.write(sortDoc, context, docWriter);+ }+ transformer.transform(doc, sortDoc.docBase + sortDoc.docId, DocIterationInfo.NONE);+ for (Map.Entry<String, Object> entry : doc) {+ if (returnFields.wantsField(entry.getKey())) {+ ew.put(entry.getKey(), entry.getValue());+ }}}- public List<FieldWriter> getFieldWriters(String[] fields, SolrQueryRequest req)+ /** Converts lazily written values to plain Java objects that can outlive the write call. */+ static Object materialize(Object v) {+ if (v instanceof IteratorWriter iteratorWriter) {+ List<Object> lst = new ArrayList<>();+ iteratorWriter.toList(lst);+ return lst;+ } else if (v instanceof MapWriter mapWriter) {+ return new SimpleOrderedMap<>(mapWriter);+ }+ return v;+ }++ List<FieldWriter> getFieldWriters(SolrReturnFields solrReturnFields, SolrQueryRequest req)throws IOException {DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false);- SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req);boolean includeStoredFields = req.getParams().getBool(INCLUDE_STORED_FIELDS_PARAM, false);+ // Also fetch fields only a transformer reads, e.g. the f= of [geo]; wantsField excludes them+ Set<String> fieldsToFetch =+ solrReturnFields.getLuceneFieldNames() != null+ ? solrReturnFields.getLuceneFieldNames()+ : Set.of();+ // Non-exportable fields named here error; ones matched only by a glob are skipped.+ // Includes fieldsToFetch, since requested names hold an alias, not its source field.+ Set<String> explicitlyNamedFields = new HashSet<>(fieldsToFetch);+ if (solrReturnFields.getRequestedFieldNames() != null) {+ explicitlyNamedFields.addAll(solrReturnFields.getRequestedFieldNames());+ }+List<FieldWriter> writers = new ArrayList<>();Set<SchemaField> docValueFields = new LinkedHashSet<>();Map<String, SchemaField> storedFields = new LinkedHashMap<>();for (String field : req.getSearcher().getFieldNames()) {- if (!solrReturnFields.wantsField(field)) {+ if (!solrReturnFields.wantsField(field) && !fieldsToFetch.contains(field)) {continue;}SchemaField schemaField = req.getSchema().getField(field);FieldType fieldType = schemaField.getType();- Set<String> requestFieldNames =- solrReturnFields.getRequestedFieldNames() == null- ? Set.of()- : solrReturnFields.getRequestedFieldNames();-if (canUseDocValues(schemaField, fieldType)) {// Prefer DocValues when availabledocValueFields.add(schemaField);@@ -517,7 +571,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {// Field is stored-only (no usable DocValues)if (includeStoredFields) {storedFields.put(field, schemaField);- } else if (requestFieldNames.contains(field)) {+ } else if (explicitlyNamedFields.contains(field)) {// Explicitly requested field without DocValues and includeStoredFields=falsethrow new IOException(schemaField@@ -525,7 +579,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {+ "Try setting includeStoredFields=true to retrieve this field from stored values.");}// Else: glob matched stored-only field without includeStoredFields - silently skip- } else if (requestFieldNames.contains(field)) {+ } else if (explicitlyNamedFields.contains(field)) {// Explicitly requested field that has neither DocValues nor storedif (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) {throw new IOException(@@ -610,6 +664,47 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {return writers;}+ /** Context for transformers in {@code fl}; there is no {@link DocList} when exporting. */+ private static final class ExportResultContext extends ResultContext {+ private final ReturnFields returnFields;+ private final SolrIndexSearcher searcher;+ private final Query query;+ private final SolrQueryRequest req;++ ExportResultContext(+ ReturnFields returnFields, SolrIndexSearcher searcher, Query query, SolrQueryRequest req) {+ this.returnFields = returnFields;+ this.searcher = searcher;+ this.query = query;+ this.req = req;+ }++ @Override+ public DocList getDocList() {+ return null;+ }++ @Override+ public ReturnFields getReturnFields() {+ return returnFields;+ }++ @Override+ public SolrIndexSearcher getSearcher() {+ return searcher;+ }++ @Override+ public Query getQuery() {+ return query;+ }++ @Override+ public SolrQueryRequest getRequest() {+ return req;+ }+ }+private static boolean canUseDocValues(SchemaField schemaField, FieldType fieldType) {return schemaField.hasDocValues()// Special handling for SortableTextField: unlike other field types, it requiresdiff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.javaindex 86bbefb8e37..3c21cb8202c 100644--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriterStream.java@@ -18,7 +18,6 @@ package org.apache.solr.handler.export;import java.io.IOException;import java.lang.invoke.MethodHandles;-import java.util.ArrayList;import java.util.List;import java.util.concurrent.BrokenBarrierException;import java.util.concurrent.TimeoutException;@@ -35,9 +34,6 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExplanation;import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParameter;import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;-import org.apache.solr.common.IteratorWriter;-import org.apache.solr.common.MapWriter;-import org.apache.solr.common.util.SimpleOrderedMap;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -65,14 +61,7 @@ public class ExportWriterStream extends TupleStream implements Expressible {@Overridepublic EntryWriter put(CharSequence k, Object v) throws IOException {- if (v instanceof IteratorWriter) {- List<Object> lst = new ArrayList<>();- ((IteratorWriter) v).toList(lst);- v = lst;- } else if (v instanceof MapWriter) {- v = new SimpleOrderedMap<>((MapWriter) v);- }- tuple.put(k.toString(), v);+ tuple.put(k.toString(), ExportWriter.materialize(v));return this;}}diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.javaindex 61384d50553..5be050c8063 100644--- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java+++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java@@ -1308,6 +1308,75 @@ public class TestExportWriter extends SolrTestCaseJ4 {jq.contains("\"sortabledv\""));}+ @Test+ @SuppressWarnings("unchecked")+ public void testDocTransformers() throws Exception {+ assertU(delQ("*:*"));+ assertU(adoc("id", "1", "intdv", "10", "intdv_m", "1", "intdv_m", "2"));+ assertU(adoc("id", "2", "intdv", "20"));+ assertU(commit());+ assertU(adoc("id", "3", "intdv", "30"));+ assertU(commit());++ Map<String, Object> selectDocIds = new HashMap<>();+ String selectRsp = h.query(req("q", "*:*", "fl", "id,[docid]", "wt", "json"));+ for (Map<String, Object> doc :+ (List<Map<String, Object>>)+ Utils.getObjectByPath(+ mapper.readValue(selectRsp, Map.class), false, "/response/docs")) {+ selectDocIds.put((String) doc.get("id"), doc.get("[docid]"));+ }+ assertEquals(3, selectDocIds.size());++ // [docid] matches /select's, and fields are materialized (incl. multiValued)+ List<Map<String, Object>> docs =+ (List<Map<String, Object>>)+ queryJsonReturnDocs(+ reqWithPath("/export", "q", "*:*", "fl", "id,intdv_m,[docid]", "sort", "id asc"));+ assertEquals(3, docs.size());+ for (Map<String, Object> doc : docs) {+ assertEquals(doc.toString(), selectDocIds.get(doc.get("id")), doc.get("[docid]"));+ }+ assertEquals(List.of(1, 2), docs.get(0).get("intdv_m"));++ // alias renames; transformer local params may contain commas+ docs =+ (List<Map<String, Object>>)+ queryJsonReturnDocs(+ reqWithPath(+ "/export", "q", "*:*", "fl", "id,x:intdv,v:[value v='a,b']", "sort", "id asc"));+ assertEquals(3, docs.size());+ assertEquals(Map.of("id", "1", "x", 10, "v", "a,b"), docs.get(0));++ // transformed fields are visible to streaming expressions+ docs =+ (List<Map<String, Object>>)+ queryJsonReturnDocs(+ reqWithPath(+ "/export",+ "q",+ "*:*",+ "fl",+ "id,[docid]",+ "sort",+ "id asc",+ "expr",+ "top(n=2,input(),sort=\"id desc\")"));+ assertEquals(2, docs.size());+ assertEquals("3", docs.get(0).get("id"));+ assertEquals(selectDocIds.get("3"), docs.get(0).get("[docid]"));++ // score is still rejected+ docs =+ (List<Map<String, Object>>)+ queryJsonReturnDocs(+ reqWithPath("/export", "q", "*:*", "fl", "id,score", "sort", "id asc"));+ assertEquals(1, docs.size());+ assertTrue(+ docs.get(0).toString(),+ docs.get(0).get(StreamParams.EXCEPTION).toString().contains("Scoring"));+ }+@SuppressWarnings("rawtypes")private void validateSort(int numDocs) throws Exception {// 10 fieldsdiff --git a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adocindex 503d12eb272..823bc51e161 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc@@ -101,6 +101,10 @@ Note that sort fields still require docValues, regardless of this setting.Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be expanded to the list of fields that match the pattern and are able to be exported, see <<Field Requirements>>.+Field aliases (e.g. `fl=id,price:price_d`) and xref:document-transformers.adoc[document transformers] (e.g. `fl=id,[docid]`) are supported, and their output is also visible to a local streaming expression.+Transformers are applied one document at a time, and those that depend on scores or on a list of matching documents won't work.+Some transformers, like `[subquery]` and `[child]`, produce output that might not work well, or at all, with `/export` and streaming expressions.+Returning scores is not supported at this time.=== Specifying the Local Streaming Expression