Apache Solr open-source search software
SOLR-9376: [xml] and [json] RawValue DocTransformers should work in cloud mode (#513)
Michael Gibney6ff81312607d
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex fbef2fff96b..abbcdbc55ca 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -266,6 +266,9 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)* SOLR-15884: Backup responses now use a map to return information instead of a list (Houston Putman, Christine Poerschke)+* SOLR-9376: Raw value DocTransformers (`[xml]`, `[json]`, `[geo w=GeoJSON]`) now work+ in a distributed/SolrCloud context (Michael Gibney)+Build---------------------* LUCENE-9077 LUCENE-9433: Support Gradle build, remove Ant support from trunk (Dawid Weiss, Erick Erickson, Uwe Schindler et.al.)diff --git a/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.javaindex 89d556bb424..c16e5b6602f 100644--- a/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/GeoJSONResponseWriter.java@@ -30,7 +30,6 @@ import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.request.SolrQueryRequest;-import org.apache.solr.response.transform.WriteableGeoJSON;import org.apache.solr.schema.AbstractSpatialFieldType;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.ReturnFields;@@ -259,11 +258,7 @@ class GeoJSONWriter extends JSONWriter {}else if(geo instanceof IndexableField) {str = ((IndexableField)geo).stringValue();- }- else if(geo instanceof WriteableGeoJSON) {- shape = ((WriteableGeoJSON)geo).shape;- }- else {+ } else {str = geo.toString();}diff --git a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.javaindex e83f9239ff6..3e5cc8bf02d 100644--- a/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/JSONResponseWriter.java@@ -203,6 +203,16 @@ class ArrayOfNameTypeValueJSONWriter extends JSONWriter {super.writeDate(name, val);}+ @Override+ public void writeStrRaw(String name, String val) throws IOException {+ if (writeTypeAndValueKey) {+ throw new IllegalStateException("NamedList should never be a field value");+ // and thus `writeTypeAndValueKey` should always have been cleared (set to false) by the+ // time `writeStrRaw(...)` is called (at the level of individual SolrDocument fields).+ }+ super.writeStrRaw(name, val);+ }+@Overridepublic void writeStr(String name, String val, boolean needsEscaping) throws IOException {ifNeededWriteTypeAndValueKey("str");@@ -230,9 +240,9 @@ class ArrayOfNameTypeValueJSONWriter extends JSONWriter {}@Override- public void writeArray(String name, Iterator<?> val) throws IOException {+ public void writeArray(String name, Iterator<?> val, boolean raw) throws IOException {ifNeededWriteTypeAndValueKey("array");- super.writeArray(name, val);+ super.writeArray(name, val, raw);}@Overridediff --git a/solr/core/src/java/org/apache/solr/response/JSONWriter.java b/solr/core/src/java/org/apache/solr/response/JSONWriter.javaindex 4e176966977..d805cf28119 100644--- a/solr/core/src/java/org/apache/solr/response/JSONWriter.java+++ b/solr/core/src/java/org/apache/solr/response/JSONWriter.java@@ -103,7 +103,7 @@ public class JSONWriter extends TextResponseWriter implements JsonTextWriter {indent();writeKey(fname, true);Object val = doc.getFieldValue(fname);- writeVal(fname, val);+ writeVal(fname, val, shouldWriteRaw(fname, returnFields));}if(doc.hasChildDocuments()) {diff --git a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.javaindex b10fb3d921f..ce2f78169ad 100644--- a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java@@ -78,8 +78,8 @@ class PHPWriter extends JSONWriter {}@Override- public void writeArray(String name, List<?> l) throws IOException {- writeArray(name,l.iterator());+ public void writeArray(String name, List<?> l, boolean raw) throws IOException {+ writeArray(name,l.iterator(), raw);}@Overridediff --git a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.javaindex 35b41af2c69..b8c5882d2e9 100644--- a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java@@ -168,7 +168,8 @@ class PHPSerializedWriter extends JSONWriter {@Override- public void writeArray(String name, Object[] val) throws IOException {+ public void writeArray(String name, Object[] val, boolean raw) throws IOException {+ assert !raw;writeMapOpener(val.length);for(int i=0; i < val.length; i++) {writeKey(i, false);@@ -178,12 +179,13 @@ class PHPSerializedWriter extends JSONWriter {}@Override- public void writeArray(String name, Iterator<?> val) throws IOException {+ public void writeArray(String name, Iterator<?> val, boolean raw) throws IOException {+ assert !raw;ArrayList<Object> vals = new ArrayList<>();while( val.hasNext() ) {vals.add(val.next());}- writeArray(name, vals.toArray());+ writeArray(name, vals.toArray(), false);}@Overridediff --git a/solr/core/src/java/org/apache/solr/response/RawShimTextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RawShimTextResponseWriter.javanew file mode 100644index 00000000000..dd7b9cc96af--- /dev/null+++ b/solr/core/src/java/org/apache/solr/response/RawShimTextResponseWriter.java@@ -0,0 +1,111 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.response;++import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.search.ReturnFields;++import java.io.IOException;+import java.util.Iterator;+import java.util.Map;++/**+ * Utility class that delegates to another {@link TextResponseWriter}, but converts normal write requests+ * into "raw" requests that write field values directly to the delegate {@link TextResponseWriter}'s backing writer.+ */+class RawShimTextResponseWriter extends TextResponseWriter {++ private final TextResponseWriter backing;++ RawShimTextResponseWriter(TextResponseWriter backing) {+ super(null, false);+ this.backing = backing;+ }++ // convert non-raw to raw. These are the reason this class exists (see class javadocs)+ @Override+ public void writeStr(String name, String val, boolean needsEscaping) throws IOException {+ backing.writeStrRaw(name, val);+ }++ @Override+ public void writeArray(String name, Iterator<?> val, boolean raw) throws IOException {+ backing.writeArray(name, val, true);+ }++ // Other stuff; just no-op delegation+ @Override+ public void writeStartDocumentList(String name, long start, int size, long numFound, Float maxScore, Boolean numFoundExact) throws IOException {+ backing.writeStartDocumentList(name, start, size, numFound, maxScore, numFoundExact);+ }++ @Override+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields fields, int idx) throws IOException {+ backing.writeSolrDocument(name, doc, fields, idx);+ }++ @Override+ public void writeEndDocumentList() throws IOException {+ backing.writeEndDocumentList();+ }++ @Override+ public void writeMap(String name, Map<?, ?> val, boolean excludeOuter, boolean isFirstVal) throws IOException {+ backing.writeMap(name, val, excludeOuter, isFirstVal);+ }++ @Override+ public void writeNull(String name) throws IOException {+ backing.writeNull(name);+ }++ @Override+ public void writeInt(String name, String val) throws IOException {+ backing.writeInt(name, val);+ }++ @Override+ public void writeLong(String name, String val) throws IOException {+ backing.writeLong(name, val);+ }++ @Override+ public void writeBool(String name, String val) throws IOException {+ backing.writeBool(name, val);+ }++ @Override+ public void writeFloat(String name, String val) throws IOException {+ backing.writeFloat(name, val);+ }++ @Override+ public void writeDouble(String name, String val) throws IOException {+ backing.writeDouble(name, val);+ }++ @Override+ public void writeDate(String name, String val) throws IOException {+ backing.writeDate(name, val);+ }++ @Override+ public void writeNamedList(String name, NamedList<?> val) throws IOException {+ backing.writeNamedList(name, val);+ }+}diff --git a/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java b/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.javaindex 7ab424d9cbc..bdc08c66b28 100644--- a/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java+++ b/solr/core/src/java/org/apache/solr/response/SchemaXmlWriter.java@@ -365,17 +365,17 @@ public class SchemaXmlWriter extends TextResponseWriter {}@Override- public void writeArray(String name, Object[] val) throws IOException {- writeArray(name, Arrays.asList(val).iterator());+ public void writeArray(String name, Object[] val, boolean raw) throws IOException {+ writeArray(name, Arrays.asList(val).iterator(), raw);}@Override- public void writeArray(String name, Iterator<?> iter) throws IOException {+ public void writeArray(String name, Iterator<?> iter, boolean raw) throws IOException {if( iter.hasNext() ) {startTag("arr", name, false );incLevel();while( iter.hasNext() ) {- writeVal(null, iter.next());+ writeVal(null, iter.next(), raw);}decLevel();if (doIndent) indent();diff --git a/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.javaindex 065d1150e58..2148107a827 100644--- a/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/TabularResponseWriter.java@@ -139,7 +139,7 @@ public abstract class TabularResponseWriter extends TextResponseWriter {}@Override- public void writeArray(String name, Iterator<?> val) throws IOException {+ public void writeArray(String name, Iterator<?> val, boolean raw) throws IOException {}@Overridediff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.javaindex e19906afc24..9ed8be90bc4 100644--- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java@@ -19,7 +19,11 @@ package org.apache.solr.response;import java.io.IOException;import java.io.Writer;import java.util.Calendar;+import java.util.Collection;+import java.util.Collections;+import java.util.HashSet;import java.util.Iterator;+import java.util.Set;import org.apache.lucene.document.Document;import org.apache.lucene.index.IndexableField;@@ -31,10 +35,12 @@ import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.FastWriter;import org.apache.solr.common.util.TextWriter;import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.transform.DocTransformer;import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.DocList;import org.apache.solr.search.ReturnFields;+import org.apache.solr.search.SolrReturnFields;/** Base class for text-oriented response writers.*@@ -55,6 +61,15 @@ public abstract class TextResponseWriter implements TextWriter {protected Calendar cal; // reusable calendar instance+ /**+ * A signal object that must be used to differentiate from <code>null</code> in strict object equality+ * checks against {@link #rawReturnFields}, in order to determine the appropriate context in which to+ * write raw field values.+ */+ private static final ReturnFields NO_RAW_FIELDS = new SolrReturnFields();+ private final TextResponseWriter rawShim;+ private final Set<String> rawFields;+ private final ReturnFields rawReturnFields;public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {this.writer = writer == null ? null: FastWriter.wrap(writer);@@ -67,6 +82,17 @@ public abstract class TextResponseWriter implements TextWriter {}returnFields = rsp.getReturnFields();if (req.getParams().getBool(CommonParams.OMIT_HEADER, false)) rsp.removeResponseHeader();+ DocTransformer rootDocTransformer = returnFields.getTransformer();+ Collection<String> rawFields;+ if (rootDocTransformer == null || (rawFields = rootDocTransformer.getRawFields()).isEmpty()) {+ this.rawFields = null;+ this.rawShim = null;+ this.rawReturnFields = NO_RAW_FIELDS;+ } else {+ this.rawFields = rawFields.size() == 1 ? Collections.singleton(rawFields.iterator().next()) : new HashSet<>(rawFields);+ this.rawShim = new RawShimTextResponseWriter(this);+ this.rawReturnFields = returnFields;+ }}//only for test purposesTextResponseWriter(Writer writer, boolean indent) {@@ -76,6 +102,16 @@ public abstract class TextResponseWriter implements TextWriter {this.rsp = null;returnFields = null;this.doIndent = indent;+ this.rawShim = null;+ this.rawFields = null;+ this.rawReturnFields = null;+ }++ /**+ * NOTE: strict object equality check against {@link #rawReturnFields}; see javadocs for {@link #NO_RAW_FIELDS}+ */+ protected final boolean shouldWriteRaw(String fname, ReturnFields returnFields) {+ return rawReturnFields == returnFields && rawFields.contains(fname);}/** done with this ResponseWriter... make sure any buffers are flushed to writer */@@ -106,7 +142,7 @@ public abstract class TextResponseWriter implements TextWriter {}- public final void writeVal(String name, Object val) throws IOException {+ public final void writeVal(String name, Object val, boolean raw) throws IOException {// if there get to be enough types, perhaps hashing on the type// to get a handler might be faster (but types must be exact to do that...)@@ -122,9 +158,10 @@ public abstract class TextResponseWriter implements TextWriter {IndexableField f = (IndexableField)val;SchemaField sf = schema.getFieldOrNull( f.name() );if( sf != null ) {- sf.getType().write(this, name, f);- }- else {+ sf.getType().write(raw ? rawShim : this, name, f);+ } else if (raw) {+ writeStrRaw(name, f.stringValue());+ } else {writeStr(name, f.stringValue(), true);}} else if (val instanceof Document) {@@ -150,7 +187,7 @@ public abstract class TextResponseWriter implements TextWriter {BytesRef arr = (BytesRef)val;writeByteArr(name, arr.bytes, arr.offset, arr.length);} else {- TextWriter.super.writeVal(name, val);+ TextWriter.super.writeVal(name, val, raw);}}// names are passed when writing primitives like writeInt to allow many differentdiff --git a/solr/core/src/java/org/apache/solr/response/XMLWriter.java b/solr/core/src/java/org/apache/solr/response/XMLWriter.javaindex e22cb8c5ec7..417938da93e 100644--- a/solr/core/src/java/org/apache/solr/response/XMLWriter.java+++ b/solr/core/src/java/org/apache/solr/response/XMLWriter.java@@ -207,7 +207,7 @@ public class XMLWriter extends TextResponseWriter {log.debug(String.valueOf(val));}}- writeVal(fname, val);+ writeVal(fname, val, shouldWriteRaw(fname, returnFields));}if(doc.hasChildDocuments()) {@@ -299,17 +299,17 @@ public class XMLWriter extends TextResponseWriter {}@Override- public void writeArray(String name, Object[] val) throws IOException {- writeArray(name, Arrays.asList(val).iterator());+ public void writeArray(String name, Object[] val, boolean raw) throws IOException {+ writeArray(name, Arrays.asList(val).iterator(), raw);}@Override- public void writeArray(String name, Iterator<?> iter) throws IOException {+ public void writeArray(String name, Iterator<?> iter, boolean raw) throws IOException {if( iter.hasNext() ) {startTag("arr", name, false );incLevel();while( iter.hasNext() ) {- writeVal(null, iter.next());+ writeVal(null, iter.next(), raw);}decLevel();if (doIndent) indent();@@ -321,7 +321,7 @@ public class XMLWriter extends TextResponseWriter {}@Override- public void writeIterator(String name, IteratorWriter val) throws IOException {+ public void writeIterator(String name, IteratorWriter val, boolean raw) throws IOException {// As the size is not known. So, always both startTag and endTag is written// irrespective of number of entries in IteratorWriterstartTag("arr", name, false );@@ -330,7 +330,7 @@ public class XMLWriter extends TextResponseWriter {val.writeIter(new IteratorWriter.ItemWriter() {@Overridepublic IteratorWriter.ItemWriter add(Object o) throws IOException {- writeVal(null, o);+ writeVal(null, o, raw);return this;}});@@ -351,6 +351,15 @@ public class XMLWriter extends TextResponseWriter {writePrim("null",name,"",false);}+ @Override+ public void writeStrRaw(String name, String val) throws IOException {+ int contentLen = val == null ? 0 : val.length();+ startTag("raw", name, contentLen == 0);+ if (contentLen == 0) return;+ writer.write(val, 0, contentLen);+ writer.write("</raw>");+ }+@Overridepublic void writeStr(String name, String val, boolean escape) throws IOException {writePrim("str",name,val,escape);diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.javaindex f1f6bf3ea37..ce329f033d8 100644--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java@@ -17,6 +17,8 @@package org.apache.solr.response.transform;import java.io.IOException;+import java.util.Collection;+import java.util.Collections;import org.apache.solr.common.SolrDocument;import org.apache.solr.response.QueryResponseWriter;@@ -52,6 +54,22 @@ public abstract class DocTransformer {}+ /**+ * If this transformer wants to bypass escaping in the {@link org.apache.solr.response.TextResponseWriter} and+ * write content directly to output for certain field(s), the names of any such field(s) should be returned+ *+ * NOTE: normally this will be conditional on the `wt` param in the request, as supplied to the+ * {@link DocTransformer}'s parent {@link TransformerFactory} at the time of transformer creation.+ *+ * @return Collection containing field names to be written raw; if no field names should+ * be written raw, an empty collection should be returned. Any collection returned collection+ * need not be externally modifiable -- i.e., {@link java.util.Collections#singleton(Object)} is+ * acceptable.+ */+ public Collection<String> getRawFields() {+ return Collections.emptySet();+ }+/*** Indicates if this transformer requires access to the underlying index to perform it's functions.*diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.javaindex 68a0b9f79f5..1bb7bc30d26 100644--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformers.java@@ -18,8 +18,10 @@ package org.apache.solr.response.transform;import java.io.IOException;import java.util.ArrayList;+import java.util.Collection;import java.util.Iterator;import java.util.List;+import java.util.stream.Collectors;import org.apache.solr.common.SolrDocument;import org.apache.solr.response.ResultContext;@@ -49,6 +51,13 @@ public class DocTransformers extends DocTransformerreturn str.toString();}+ @Override+ public Collection<String> getRawFields() {+ return children.stream().map(DocTransformer::getRawFields)+ .flatMap(Collection::stream)+ .collect(Collectors.toList());+ }+public void addTransformer( DocTransformer a ) {children.add( a );}diff --git a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.javaindex 790d93899f9..3c4dc53539a 100644--- a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java+++ b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java@@ -17,7 +17,11 @@package org.apache.solr.response.transform;import java.io.IOException;+import java.util.Collection;+import java.util.Collections;import java.util.Iterator;+import java.util.Map;+import java.util.Set;import org.apache.lucene.index.IndexableField;import org.apache.lucene.index.LeafReaderContext;@@ -63,10 +67,15 @@ import org.locationtech.spatial4j.shape.Shape;* </ul>**/-public class GeoTransformerFactory extends TransformerFactory-{+public class GeoTransformerFactory extends TransformerFactory implements TransformerFactory.FieldRenamer {+@Overridepublic DocTransformer create(String display, SolrParams params, SolrQueryRequest req) {+ throw new UnsupportedOperationException();+ }++ @Override+ public DocTransformer create(String display, SolrParams params, SolrQueryRequest req, Map<String, String> renamedFields, Set<String> reqFieldNames) {String fname = params.get("f", display);if(fname.startsWith("[") && fname.endsWith("]")) {@@ -124,12 +133,7 @@ public class GeoTransformerFactory extends TransformerFactory// Using ValueSourceif(shapes!=null) {- return new DocTransformer() {- @Override- public String getName() {- return display;- }-+ return new GeoDocTransformer(updater) {@Overridepublic void transform(SolrDocument doc, int docid) throws IOException {int leafOrd = ReaderUtil.subIndex(docid, context.getSearcher().getTopReaderContext().leaves());@@ -144,22 +148,26 @@ public class GeoTransformerFactory extends TransformerFactory}+ // if source has been renamed, update reference+ updater.field = renamedFields.getOrDefault(updater.field, updater.field);++ // don't remove fields that were explicitly requested by others+ final boolean copy = reqFieldNames != null && reqFieldNames.contains(updater.field);+ if (!copy) {+ renamedFields.put(updater.field, updater.display);+ }+// Using the raw stored values- return new DocTransformer() {+ return new GeoDocTransformer(updater) {@Overridepublic void transform(SolrDocument doc, int docid) throws IOException {- Object val = doc.remove(updater.field);+ Object val = copy ? doc.get(updater.field) : doc.remove(updater.field);if(val!=null) {updater.setValue(doc, val);}}- @Override- public String getName() {- return updater.display;- }-@Overridepublic String[] getExtraRequestFields() {return new String[] {updater.field};@@ -167,65 +175,77 @@ public class GeoTransformerFactory extends TransformerFactory};}-}+ private static abstract class GeoDocTransformer extends DocTransformer {-class GeoFieldUpdater {- String field;- String display;- String display_error;-- boolean isJSON;- ShapeWriter writer;- SupportedFormats formats;-- void addShape(SolrDocument doc, Shape shape) {- if(isJSON) {- doc.addField(display, new WriteableGeoJSON(shape, writer));+ private final GeoFieldUpdater updater;++ private GeoDocTransformer(GeoFieldUpdater updater) {+ this.updater = updater;}- else {- doc.addField(display, writer.toString(shape));++ @Override+ public String getName() {+ return updater.display;}- }-- void setValue(SolrDocument doc, Object val) {- doc.remove(display);- if(val != null) {- if(val instanceof Iterable) {- Iterator<?> iter = ((Iterable<?>)val).iterator();- while(iter.hasNext()) {- addValue(doc, iter.next());- }- }- else {- addValue(doc, val);- }++ @Override+ public Collection<String> getRawFields() {+ return updater.isJSON ? Collections.singleton(updater.display) : Collections.emptySet();}}-- void addValue(SolrDocument doc, Object val) {- if(val == null) {- return;- }-- if(val instanceof Shape) {- addShape(doc, (Shape)val);++ private static class GeoFieldUpdater {+ String field;+ String display;+ String display_error;++ boolean isJSON;+ ShapeWriter writer;+ SupportedFormats formats;++ void addShape(SolrDocument doc, Shape shape) {+ doc.addField(display, writer.toString(shape));}- // Don't explode on 'InvalidShpae'- else if( val instanceof Exception) {- doc.setField( display_error, ((Exception)val).toString() );++ void setValue(SolrDocument doc, Object val) {+ doc.remove(display);+ if(val != null) {+ if(val instanceof Iterable) {+ Iterator<?> iter = ((Iterable<?>)val).iterator();+ while(iter.hasNext()) {+ addValue(doc, iter.next());+ }+ }+ else {+ addValue(doc, val);+ }+ }}- else {- // Use the stored value- if(val instanceof IndexableField) {- val = ((IndexableField)val).stringValue();++ void addValue(SolrDocument doc, Object val) {+ if(val == null) {+ return;+ }++ if(val instanceof Shape) {+ addShape(doc, (Shape)val);}- try {- addShape(doc, formats.read(val.toString()));+ // Don't explode on 'InvalidShpae'+ else if( val instanceof Exception) {+ doc.setField( display_error, ((Exception)val).toString() );}- catch(Exception ex) {- doc.setField( display_error, ex.toString() );+ else {+ // Use the stored value+ if(val instanceof IndexableField) {+ val = ((IndexableField)val).stringValue();+ }+ try {+ addShape(doc, formats.read(val.toString()));+ }+ catch(Exception ex) {+ doc.setField( display_error, ex.toString() );+ }}}}}-diff --git a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.javaindex 5838e10520e..da657bb9c95 100644--- a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java+++ b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java@@ -16,27 +16,23 @@*/package org.apache.solr.response.transform;-import java.io.IOException;-import java.util.ArrayList;import java.util.Collection;+import java.util.Collections;+import java.util.Map;+import java.util.Set;import com.google.common.base.Strings;-import org.apache.lucene.index.IndexableField;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.JavaBinCodec;-import org.apache.solr.common.util.JavaBinCodec.ObjectResolver;import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.TextWriter;-import org.apache.solr.common.util.WriteableValue;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.QueryResponseWriter;/*** @since solr 5.2*/-public class RawValueTransformerFactory extends TransformerFactory+public class RawValueTransformerFactory extends TransformerFactory implements TransformerFactory.FieldRenamer{String applyToWT = null;@@ -58,10 +54,28 @@ public class RawValueTransformerFactory extends TransformerFactory@Overridepublic DocTransformer create(String display, SolrParams params, SolrQueryRequest req) {+ throw new UnsupportedOperationException();+ }++ @Override+ public boolean mayModifyValue() {+ // The only thing we may modify is the _serialization_; field values per se are guaranteed to be unmodified.+ return false;+ }++ @Override+ public DocTransformer create(String display, SolrParams params, SolrQueryRequest req,+ Map<String, String> renamedFields, Set<String> reqFieldNames) {String field = params.get("f");if(Strings.isNullOrEmpty(field)) {field = display;}+ field = renamedFields.getOrDefault(field, field);+ final boolean rename = !field.equals(display);+ final boolean copy = rename && reqFieldNames != null && reqFieldNames.contains(field);+ if (!copy) {+ renamedFields.put(field, display);+ }// When a 'wt' is specified in the transformer, only apply it to the same wtboolean apply = true;if(applyToWT!=null) {@@ -79,25 +93,27 @@ public class RawValueTransformerFactory extends TransformerFactory}if(apply) {- return new RawTransformer( field, display );+ return new RawTransformer( field, display, copy );}- if (field.equals(display)) {+ if (!rename) {// we have to ensure the field is returnedreturn new DocTransformer.NoopFieldTransformer(field);}- return new RenameFieldTransformer( field, display, false );+ return new RenameFieldTransformer( field, display, copy );}-+static class RawTransformer extends DocTransformer{final String field;final String display;+ final boolean copy;- public RawTransformer( String field, String display )+ public RawTransformer( String field, String display, boolean copy ){this.field = field;this.display = display;+ this.copy = copy;}@Override@@ -107,57 +123,21 @@ public class RawValueTransformerFactory extends TransformerFactory}@Override- public void transform(SolrDocument doc, int docid) {- Object val = doc.remove(field);- if(val==null) {- return;- }- if(val instanceof Collection) {- Collection<?> current = (Collection<?>)val;- ArrayList<WriteableStringValue> vals = new ArrayList<RawValueTransformerFactory.WriteableStringValue>();- for(Object v : current) {- vals.add(new WriteableStringValue(v));- }- doc.setField(display, vals);- }- else {- doc.setField(display, new WriteableStringValue(val));- }+ public Collection<String> getRawFields() {+ return Collections.singleton(display);}@Override- public String[] getExtraRequestFields() {- return new String[] {this.field};- }- }-- public static class WriteableStringValue extends WriteableValue {- public final Object val;-- public WriteableStringValue(Object val) {- this.val = val;- }-- @Override- public void write(String name, TextWriter writer) throws IOException {- String str = null;- if(val instanceof IndexableField) { // delays holding it in memory- str = ((IndexableField)val).stringValue();- }- else {- str = val.toString();+ public void transform(SolrDocument doc, int docid) {+ Object val = copy ? doc.get(field) : doc.remove(field);+ if(val != null) {+ doc.setField(display, val);}- writer.getWriter().write(str);}@Override- public Object resolve(Object o, JavaBinCodec codec) throws IOException {- ObjectResolver orig = codec.getResolver();- if(orig != null) {- codec.writeVal(orig.resolve(val, codec));- return null;- }- return val.toString();+ public String[] getExtraRequestFields() {+ return new String[] {this.field};}}}diff --git a/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.javaindex 41ac54f31dc..d67ea410046 100644--- a/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java+++ b/solr/core/src/java/org/apache/solr/response/transform/RenameFieldTransformer.java@@ -29,12 +29,14 @@ public class RenameFieldTransformer extends DocTransformerfinal String from;final String to;final boolean copy;+ final String[] ensureFromFieldPresent;- public RenameFieldTransformer( String from, String to, boolean copy )- {+ public RenameFieldTransformer( String from, String to, boolean copy ) {this.from = from;this.to = to;this.copy = copy;+ this.ensureFromFieldPresent = new String[] { from };+ assert !from.equals(to);}@Override@@ -50,4 +52,9 @@ public class RenameFieldTransformer extends DocTransformerdoc.setField(to, v);}}++ @Override+ public String[] getExtraRequestFields() {+ return ensureFromFieldPresent;+ }}diff --git a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.javaindex 97134ea71a5..ab9807f3683 100644--- a/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java+++ b/solr/core/src/java/org/apache/solr/response/transform/TransformerFactory.java@@ -18,6 +18,7 @@ package org.apache.solr.response.transform;import java.util.HashMap;import java.util.Map;+import java.util.Set;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;@@ -40,6 +41,62 @@ public abstract class TransformerFactory implements NamedListInitializedPluginpublic abstract DocTransformer create(String field, SolrParams params, SolrQueryRequest req);+ /**+ * The {@link FieldRenamer} interface should be implemented by any {@link TransformerFactory} capable of generating+ * transformers that might rename fields, and should implement {@link #create(String, SolrParams, SolrQueryRequest, Map, Set)}+ * in place of {@link #create(String, SolrParams, SolrQueryRequest)} (with the latter method+ * overridden to throw {@link UnsupportedOperationException}).+ *+ * {@link DocTransformer}s returned via {@link #create(String, SolrParams, SolrQueryRequest, Map, Set)}+ * will be added in a second pass, allowing simplified logic in {@link TransformerFactory#create(String, SolrParams, SolrQueryRequest)}+ * for non-renaming factories.+ *+ * {@link #create(String, SolrParams, SolrQueryRequest, Map, Set)} must implement extra logic to be aware of+ * preceding field renames, and to make subsequent {@link FieldRenamer} transformers aware of its own field renames.+ *+ * It is harmless for a {@link DocTransformer} that does _not_ in practice rename fields to be returned from a+ * factory that implements this interface (e.g., for conditional renames?); but doing so opens the possibility of+ * {@link #create(String, SolrParams, SolrQueryRequest, Map, Set)} being called _after_ fields have been renamed,+ * so such implementations must still check whether the field with which they are concerned has been renamed ...+ * and if it _has_, must copy the field back to its original name. This situation also demonstrates the+ * motivation for separating the creation of {@link DocTransformer}s into two phases: an initial phase involving+ * no field renames, and a subsequent phase that implement extra logic to properly handle field renames.+ */+ public interface FieldRenamer {+ // TODO: Behavior is undefined in the event of a "destination field" collision (e.g., a user maps two fields to+ // the same "destination field", or maps a field to a top-level requested field). In the future, the easiest way+ // to detect such a case would be by "failing fast" upon renaming to a field that already has an associated value,+ // or support for this feature could be expressly added via a hypothetical+ // `combined_field:[consolidate fl=field_1,field_2]` transformer.+ /**+ * Analogous to {@link TransformerFactory#create(String, SolrParams, SolrQueryRequest)}, but to be implemented+ * by {@link TransformerFactory}s that produce {@link DocTransformer}s that may rename fields.+ *+ * @param field The destination field+ * @param params Local params associated with this transformer (e.g., source field)+ * @param req The current request+ * @param renamedFields Maps source=>dest renamed fields. Implementations should check this first, updating+ * their own "source" field(s) as necessary, and if renaming (not copying) fields, should+ * also update this map with the implementations "own" introduced source=>dest field+ * mapping+ * @param reqFieldNames Set of explicitly requested field names; implementations should consult this set to+ * determine whether it's appropriate to rename (vs. copy) a field (e.g.: <code>boolean+ * copy = reqFieldNames != null && reqFieldNames.contains(sourceField)</code>)+ * @return A transformer to be used in processing field values in returned documents.+ */+ DocTransformer create(String field, SolrParams params, SolrQueryRequest req, Map<String, String> renamedFields, Set<String> reqFieldNames);++ /**+ * Returns <code>true</code> if implementations of this class may (even subtly) modify field values.+ * ({@link GeoTransformerFactory} may do this, e.g.). To fail safe, the default implementation returns+ * <code>true</code>. This method should be overridden to return <code>false</code> if the implementing+ * class is guaranteed to not modify any values for the fields that it renames.+ */+ default boolean mayModifyValue() {+ return true;+ }+ }+public static final Map<String,TransformerFactory> defaultFactories = new HashMap<>(9, 1.0f);static {defaultFactories.put( "explain", new ExplainAugmenterFactory() );diff --git a/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java b/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.javadeleted file mode 100644index f89f5c847f5..00000000000--- a/solr/core/src/java/org/apache/solr/response/transform/WriteableGeoJSON.java+++ /dev/null@@ -1,55 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.response.transform;--import java.io.IOException;--import org.apache.solr.common.util.JavaBinCodec;-import org.apache.solr.common.util.TextWriter;-import org.apache.solr.common.util.WriteableValue;-import org.locationtech.spatial4j.io.ShapeWriter;-import org.locationtech.spatial4j.shape.Shape;--/**- * This will let the writer add values to the response directly- */-public class WriteableGeoJSON extends WriteableValue {-- public final Shape shape;- public final ShapeWriter jsonWriter;-- public WriteableGeoJSON(Shape shape, ShapeWriter jsonWriter) {- this.shape = shape;- this.jsonWriter = jsonWriter;- }-- @Override- public Object resolve(Object o, JavaBinCodec codec) throws IOException {- codec.writeStr(jsonWriter.toString(shape));- return null; // this means we wrote it- }-- @Override- public void write(String name, TextWriter writer) throws IOException {- jsonWriter.write(writer.getWriter(), shape);- }-- @Override- public String toString() {- return jsonWriter.toString(shape);- }-}diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.javaindex 6135bb245e0..4903e0b18b9 100644--- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java+++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java@@ -16,9 +16,12 @@*/package org.apache.solr.search;+import java.util.ArrayDeque;import java.util.ArrayList;import java.util.Collection;import java.util.Collections;+import java.util.Deque;+import java.util.HashMap;import java.util.HashSet;import java.util.LinkedHashSet;import java.util.List;@@ -35,7 +38,6 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.NamedList;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.transform.DocTransformer;import org.apache.solr.response.transform.DocTransformers;@@ -160,6 +162,10 @@ public class SolrReturnFields extends ReturnFields {}+ /**+ * Parsing is done in two passes (see javadocs for {@link org.apache.solr.response.transform.TransformerFactory.FieldRenamer}+ * for an explanation of the logic behind deferring creation of "rename field" transformers).+ */private void parseFieldList(String[] fl, SolrQueryRequest req) {_wantsScore = false;_wantsAllFields = false;@@ -168,32 +174,26 @@ public class SolrReturnFields extends ReturnFields {return;}- NamedList<String> rename = new NamedList<>();+ Deque<DeferredRenameEntry> deferredRenameAugmenters = new ArrayDeque<>();DocTransformers augmenters = new DocTransformers();for (String fieldList : fl) {- add(fieldList,rename,augmenters,req);+ add(fieldList,deferredRenameAugmenters,augmenters,req);}- for( int i=0; i<rename.size(); i++ ) {- String from = rename.getName(i);- String to = rename.getVal(i);- okFieldNames.add( to );- boolean copy = (reqFieldNames!=null && reqFieldNames.contains(from));- if(!copy) {- // Check that subsequent copy/rename requests have the field they need to copy- for(int j=i+1; j<rename.size(); j++) {- if(from.equals(rename.getName(j))) {- rename.setName(j, to); // copy from the current target- if(reqFieldNames==null) {- reqFieldNames = new LinkedHashSet<>();- }- reqFieldNames.add(to); // don't rename our current target+ Map<String, String> renamedNotCopied = new HashMap<>();+ for (DeferredRenameEntry e : deferredRenameAugmenters) {+ DocTransformer t = e.create(renamedNotCopied, reqFieldNames);+ augmenters.addTransformer(t);+ if (!_wantsAllFields) {+ final String[] extraRequestFields = t.getExtraRequestFields();+ if (extraRequestFields != null) {+ for (String f : extraRequestFields) {+ fields.add(f);}}}- augmenters.addTransformer( new RenameFieldTransformer( from, to, copy ) );}- if (rename.size() > 0 ) {- renameFields = rename.asShallowMap();+ if (!renamedNotCopied.isEmpty()) {+ renameFields = renamedNotCopied;}if( !_wantsAllFields && !globs.isEmpty() ) {// TODO??? need to fill up the fields with matching field names in the index@@ -236,7 +236,7 @@ public class SolrReturnFields extends ReturnFields {return null;}- private void add(String fl, NamedList<String> rename, DocTransformers augmenters, SolrQueryRequest req) {+ private void add(String fl, Deque<DeferredRenameEntry> deferred, DocTransformers augmenters, SolrQueryRequest req) {if( fl == null ) {return;}@@ -278,8 +278,9 @@ public class SolrReturnFields extends ReturnFields {field = sp.getId(null);ch = sp.ch();if (field != null && (Character.isWhitespace(ch) || ch == ',' || ch==0)) {- rename.add(field, key);- addField(field, key, augmenters, false);+ deferred.addFirst(new DeferredRenameEntry(key, new ModifiableSolrParams().set(SOURCE_FIELD_ARGNAME, field), req, RENAME_FIELD_TRANSFORMER_FACTORY));+ // NOTE: treat as pseudoField below because `fields` will be modified on deferred invocation+ addField(field, key, augmenters, true);continue;}// an invalid field name... reset the position pointer to retry@@ -326,7 +327,20 @@ public class SolrReturnFields extends ReturnFields {}TransformerFactory factory = req.getCore().getTransformerFactory( augmenterName );- if( factory != null ) {+ if (factory instanceof TransformerFactory.FieldRenamer) {+ // NOTE: `deferred` is a Deque because some TransformerFactories (e.g., `GeoTransformerFactory`) can+ // subtly modify the representation of the associated value (i.e., it's not just a straight rename). This+ // subverts the "update source field" phase of `FieldRenamer.create(...)`. We _know_ however that "simple"+ // field renames don't do any value modification whatsoever, so those are added to the beginning of+ // the `deferred` Deque so that they will be processed first, and all other `FieldRenamers` are+ // added (here) to the front or back of the Deque, depending on the return value of `mayModifyValue()`.+ final DeferredRenameEntry deferredEntry = new DeferredRenameEntry(disp, augmenterParams, req, (TransformerFactory.FieldRenamer) factory);+ if (((TransformerFactory.FieldRenamer) factory).mayModifyValue()) {+ deferred.addLast(deferredEntry);+ } else {+ deferred.addFirst(deferredEntry);+ }+ } else if (factory != null) {DocTransformer t = factory.create(disp, augmenterParams, req);if(t!=null) {if(!_wantsAllFields) {@@ -415,7 +429,7 @@ public class SolrReturnFields extends ReturnFields {// OK, it was an oddly named fieldaddField(field, key, augmenters, false);if( key != null ) {- rename.add(field, key);+ deferred.addFirst(new DeferredRenameEntry(key, new ModifiableSolrParams().set(SOURCE_FIELD_ARGNAME, field), req, RENAME_FIELD_TRANSFORMER_FACTORY));}} else {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error parsing fieldname: " + e.getMessage(), e);@@ -430,6 +444,43 @@ public class SolrReturnFields extends ReturnFields {}}+ private static final String SOURCE_FIELD_ARGNAME = "sourceField";+ private static final TransformerFactory.FieldRenamer RENAME_FIELD_TRANSFORMER_FACTORY = new TransformerFactory.FieldRenamer() {+ @Override+ public DocTransformer create(String to, SolrParams params, SolrQueryRequest req, Map<String, String> renamedFields, Set<String> reqFieldNames) {+ String from = params.get(SOURCE_FIELD_ARGNAME);+ from = renamedFields.getOrDefault(from, from);+ final boolean copy = reqFieldNames != null && reqFieldNames.contains(from);+ if (!copy) {+ renamedFields.put(from, to);+ }+ return new RenameFieldTransformer(from, to, copy);+ }++ @Override+ public boolean mayModifyValue() {+ return false;+ }+ };++ private static final class DeferredRenameEntry {+ private final String field;+ private final SolrParams params;+ private final SolrQueryRequest req;+ private final TransformerFactory.FieldRenamer factory;++ private DeferredRenameEntry(String field, SolrParams params, SolrQueryRequest req, TransformerFactory.FieldRenamer factory) {+ this.field = field;+ this.params = params;+ this.req = req;+ this.factory = factory;+ }++ private DocTransformer create(Map<String, String> renamedFields, Set<String> reqFieldNames) {+ return factory.create(field, params, req, renamedFields, reqFieldNames);+ }+ }+private void addField(String field, String key, DocTransformers augmenters, boolean isPseudoField){if(reqFieldNames==null) {diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.javaindex 34b02793b3c..26b2fdf94c3 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java@@ -17,6 +17,7 @@package org.apache.solr.cloud;import java.io.IOException;+import java.io.StringReader;import java.lang.invoke.MethodHandles;import java.nio.file.Path;import java.util.ArrayList;@@ -33,16 +34,21 @@ import java.util.TreeSet;import org.apache.commons.io.FilenameUtils;import org.apache.lucene.util.TestUtil;+import org.apache.solr.client.solrj.ResponseParser;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.NoOpResponseParser;+import org.apache.solr.client.solrj.impl.XMLResponseParser;import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;@@ -52,9 +58,14 @@ import org.apache.solr.response.transform.TransformerFactory;import org.apache.solr.util.RandomizeSSL;import org.junit.AfterClass;import org.junit.BeforeClass;+import org.noggit.ObjectBuilder;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import javax.xml.stream.XMLStreamConstants;+import javax.xml.stream.XMLStreamException;+import javax.xml.stream.XMLStreamReader;+/** @see TestCloudPseudoReturnFields */@RandomizeSSL(clientAuth=0.0,reason="client auth uses too much RAM")public class TestRandomFlRTGCloud extends SolrCloudTestCase {@@ -91,18 +102,28 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {new ValueAugmenterValidator(1976, "val_alias"),//new RenameFieldValueValidator("id", "my_id_alias"),+ // NOTE: we add a SimpleFieldValueValidator below to check that we can enforce the presence of this field,+ // even when it may have been "renamed" by the transformer above? (this and other such instances are+ // marked with `//REQ`); also add a RenameFieldValueValidator to "fork" values, marked with `//FORK`.+ new SimpleFieldValueValidator("id"), //REQnew SimpleFieldValueValidator("aaa_i"),new RenameFieldValueValidator("bbb_i", "my_int_field_alias"),+ new RenameFieldValueValidator("bbb_i", "my_int_field_alias2"), //FORK+ new SimpleFieldValueValidator("bbb_i"), //REQnew SimpleFieldValueValidator("ccc_s"),new RenameFieldValueValidator("ddd_s", "my_str_field_alias"),- //- // SOLR-9376: RawValueTransformerFactory doesn't work in cloud mode- //- // new RawFieldValueValidator("json", "eee_s", "my_json_field_alias"),- // new RawFieldValueValidator("json", "fff_s"),- // new RawFieldValueValidator("xml", "ggg_s", "my_xml_field_alias"),- // new RawFieldValueValidator("xml", "hhh_s"),- //+ new RenameFieldValueValidator("ddd_s", "my_str_field_alias2"), // FORK+ new SimpleFieldValueValidator("ddd_s"), //REQ++ new RawFieldValueValidator("json", "eee_s", "my_json_field_alias"),+ new RenameFieldValueValidator("eee_s", "my_escaped_json_field_alias"), // FORK+ new SimpleFieldValueValidator("eee_s"), //REQ+ new RawFieldValueValidator("json", "fff_s"),+ new RawFieldValueValidator("xml", "ggg_s", "my_xml_field_alias"),+ new RenameFieldValueValidator("ggg_s", "my_escaped_xml_field_alias"), // FORK+ new SimpleFieldValueValidator("ggg_s"), //REQ+ new RawFieldValueValidator("xml", "hhh_s"),+new NotIncludedValidator("bogus_unused_field_ss"),new NotIncludedValidator("bogus_alias","bogus_alias:other_bogus_field_i"),new NotIncludedValidator("bogus_raw_alias","bogus_raw_alias:[xml f=bogus_raw_field_ss]"),@@ -111,6 +132,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {new FunctionValidator("aaa_i", "func_aaa_alias"),new GeoTransformerValidator("geo_1_srpt"),new GeoTransformerValidator("geo_2_srpt","my_geo_alias"),+ new RenameFieldValueValidator("geo_2_srpt", "my_geo_alias2"), // FORK+ new SimpleFieldValueValidator("geo_2_srpt"), //REQnew ExplainValidator(),new ExplainValidator("explain_alias"),new SubQueryValidator(),@@ -144,7 +167,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {.withProperty("schema", "schema-pseudo-fields.xml").process(CLOUD_CLIENT);- cluster.waitForActiveCollection(COLLECTION_NAME, numShards, repFactor * numShards);+ cluster.waitForActiveCollection(COLLECTION_NAME, numShards, repFactor * numShards);for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));@@ -187,7 +210,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {// items should only be added to this list if it's known that they do not work with RTG// and a specific Jira for fixing this is listed as a commentfinal List<String> knownBugs = Arrays.asList- ( "xml","json", // SOLR-9376+ ("child" // way to complicatd to vet with this test, see SOLR-9379 instead);@@ -318,10 +341,10 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {//"ccc_s", TestUtil.randomSimpleString(random()),"ddd_s", TestUtil.randomSimpleString(random()),- "eee_s", TestUtil.randomSimpleString(random()),- "fff_s", TestUtil.randomSimpleString(random()),- "ggg_s", TestUtil.randomSimpleString(random()),- "hhh_s", TestUtil.randomSimpleString(random()),+ "eee_s", makeJson(TestUtil.randomSimpleString(random())),+ "fff_s", makeJson(TestUtil.randomSimpleString(random())),+ "ggg_s", makeXml(TestUtil.randomSimpleString(random())),+ "hhh_s", makeXml(TestUtil.randomSimpleString(random())),//"geo_1_srpt", GeoTransformerValidator.getValueForIndexing(random()),"geo_2_srpt", GeoTransformerValidator.getValueForIndexing(random()),@@ -338,7 +361,60 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {return doc;}-+ private String makeJson(String s) {+ switch (random().nextInt(3)) {+ case 0:+ // simple string+ return '"' + s + '"';+ case 1:+ // array+ return "[\"" + s + "\", \"" + s + "\"]";+ case 2:+ // map+ return "{\"" + s + "\":\"" + s + "\"}";+ default:+ throw new IllegalStateException();+ }+ }++ private String makeXml(String s) {+ switch (random().nextInt(3)) {+ case 0:+ // simple string+ return s;+ case 1:+ // simple element+ return "<root>" + s + "</root>";+ case 2:+ // slightly more complex+ return "<root><inner1>" + s + "</inner1><inner2>" + s + "</inner2></root>";+ default:+ throw new IllegalStateException();+ }+ }++ private static final ResponseParser RAW_XML_RESPONSE_PARSER = new NoOpResponseParser();+ private static final ResponseParser RAW_JSON_RESPONSE_PARSER = new NoOpResponseParser() {+ @Override+ public String getWriterType() {+ return "json";+ }+ };++ private static ResponseParser modifyParser(HttpSolrClient client, final String wt) {+ final ResponseParser ret = client.getParser();+ switch (wt) {+ case "xml":+ client.setParser(RAW_XML_RESPONSE_PARSER);+ return ret;+ case "json":+ client.setParser(RAW_JSON_RESPONSE_PARSER);+ return ret;+ default:+ return null;+ }+ }+/*** Does one or more RTG request for the specified docIds with a randomized fl & fq params, asserting* that the returned document (if any) makes sense given the expected SolrInputDocuments@@ -397,11 +473,44 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {assert 1 == idsToRequest.size();params.add("id",idsToRequest.get(0));}-- final QueryResponse rsp = client.query(params);- assertNotNull(params.toString(), rsp);- final SolrDocumentList docs = getDocsFromRTGResponse(askForList, rsp);+ String wt = params.get(CommonParams.WT, "javabin");+ final ResponseParser restoreResponseParser;+ if (client instanceof HttpSolrClient) {+ restoreResponseParser = modifyParser((HttpSolrClient) client, wt);+ } else {+ // unless HttpSolrClient, `wt` doesn't matter -- it'll always be binary.+ wt = "javabin";+ restoreResponseParser = null;+ }++ final Object rsp;+ final SolrDocumentList docs;+ if ("javabin".equals(wt)) {+ // the most common case+ final QueryResponse qRsp = client.query(params);+ assertNotNull(params.toString(), qRsp);+ rsp = qRsp;+ docs = getDocsFromRTGResponse(askForList, qRsp);+ } else {+ final NamedList<Object> nlRsp = client.request(new QueryRequest(params));+ assertNotNull(restoreResponseParser);+ ((HttpSolrClient) client).setParser(restoreResponseParser);+ assertNotNull(params.toString(), nlRsp);+ rsp = nlRsp;+ final String textResult = (String) nlRsp.get("response");+ switch (wt) {+ case "json":+ docs = getDocsFromJsonResponse(askForList, textResult);+ break;+ case "xml":+ docs = getDocsFromXmlResponse(askForList, textResult);+ break;+ default:+ throw new IllegalStateException();+ }+ }+assertNotNull(params + " => " + rsp, docs);assertEquals("num docs mismatch: " + params + " => " + docsToExpect + " vs " + docs,@@ -416,7 +525,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {Set<String> expectedFieldNames = new TreeSet<>();for (FlValidator v : validators) {- expectedFieldNames.addAll(v.assertRTGResults(validators, expected, actual));+ expectedFieldNames.addAll(v.assertRTGResults(validators, expected, actual, wt));}// ensure only expected field names are in the actual documentSet<String> actualFieldNames = new TreeSet<>(actual.getFieldNames());@@ -450,8 +559,36 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {}return result;}-- /**++ @SuppressWarnings("unchecked")+ private static SolrDocumentList getSolrDocumentList(Map<String, Object> response) {+ SolrDocumentList ret = new SolrDocumentList();+ for (Map<String, Object> doc : (List<Map<String, Object>>) response.get("docs")) {+ ret.add(new SolrDocument(doc));+ }+ return ret;+ }++ @SuppressWarnings("unchecked")+ private static SolrDocumentList getDocsFromJsonResponse(final boolean expectList, final String rsp) throws IOException {+ Map<String, Object> nl = (Map<String, Object>) ObjectBuilder.fromJSON(rsp);+ if (expectList) {+ return getSolrDocumentList((Map<String, Object>) nl.get("response"));+ } else {+ SolrDocumentList ret = new SolrDocumentList();+ Map<String, Object> doc = (Map<String, Object>) nl.get("doc");+ if (doc != null) {+ ret.add(new SolrDocument(doc));+ }+ return ret;+ }+ }++ private static SolrDocumentList getDocsFromXmlResponse(final boolean expectList, final String rsp) {+ return getDocsFromRTGResponse(expectList, new QueryResponse(new RawCapableXMLResponseParser().processResponse(new StringReader(rsp)), null));+ }++ /*** returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed* at a node in our cluster*/@@ -531,11 +668,13 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {* @param validators all validators in use for this request, including the current one* @param expected a document containing the expected fields & values that should be in the index* @param actual A document that was returned by an RTG request+ * @param wt the `wt` serialization of the response* @return A set of "field names" in the actual document that this validator expected.*/public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual);+ final SolrDocument actual,+ final String wt);}/**@@ -554,15 +693,35 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {this.actualFieldName = actualFieldName;}public abstract String getFlParam();++ @Overridepublic Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {assertEquals(expectedFieldName + " vs " + actualFieldName,- expected.getFieldValue(expectedFieldName), actual.getFirstValue(actualFieldName));+ expected.getFieldValue(expectedFieldName), normalize(wt, actual.getFirstValue(actualFieldName)));return Collections.<String>singleton(actualFieldName);}}-++ /**+ * Json parsing results in all Long and Double number values; `expected` values are all (conveniently!)+ * expressed as Integer and Float, so we do a little normalization here so that the values are compatible+ */+ private static Object normalize(String wt, Object val) {+ if ("json".equals(wt) && val instanceof Number) {+ if (val instanceof Long) {+ return ((Long) val).intValue();+ } else if (val instanceof Double) {+ return ((Double) val).floatValue();+ } else {+ throw new IllegalStateException("numbers with `wt=json` only expect Long or Double");+ }+ }+ return val;+ }+private static class SimpleFieldValueValidator extends FieldValueValidator {public SimpleFieldValueValidator(final String fieldName) {super(fieldName, fieldName);@@ -588,15 +747,16 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {* What we're primarily concerned with is that the transformer does it's job and puts the string* in the response, regardless of cloud/RTG/uncommited state of the document.*/- @SuppressWarnings("UnusedNestedClass") // SOLR-9376private static class RawFieldValueValidator extends RenameFieldValueValidator {final String type;final String alias;+ final SolrParams extraParams;public RawFieldValueValidator(final String type, final String fieldName, final String alias) {// transformer is weird, default result key doesn't care what params are used...super(fieldName, null == alias ? "["+type+"]" : alias);this.type = type;this.alias = alias;+ this.extraParams = new ModifiableSolrParams().set(CommonParams.WT, type);}public RawFieldValueValidator(final String type, final String fieldName) {this(type, fieldName, null);@@ -604,11 +764,96 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() {return (null == alias ? "" : (alias + ":")) + "[" + type + " f=" + expectedFieldName + "]";}+ @Override+ public Collection<String> assertRTGResults(final Collection<FlValidator> validators,+ SolrInputDocument expected,+ final SolrDocument actual,+ final String wt) {+ if ("json".equals(wt) && "json".equals(type)) {+ Object v = actual.get(actualFieldName);+ if (v instanceof Collection) {+ // the json "array" type is indistinguishable from a multivalued field, so when `super` validates+ // based on `actual.getFirstValue(...)`, it causes issues. Here we know that our raw values are only+ // on single-valued fields, so we wrap it to work around `getFirstValue` in parent class.+ // The same logic applies to `expected` (below)+ actual.setField(actualFieldName, Collections.singleton(v));+ }+ try {+ Object parsedExpected = ObjectBuilder.fromJSON((String) expected.getFieldValue(expectedFieldName));+ if (parsedExpected instanceof Collection) {+ // see note above+ parsedExpected = Collections.singleton(parsedExpected);+ }+ expected = expected.deepCopy(); // need to copy before modifying expected!+ expected.setField(expectedFieldName, parsedExpected);+ } catch (IOException ex) {+ // swallow the exception and use the un-parsed String?+ }+ } else if ("xml".equals(wt) && "xml".equals(type)) {+ try {+ Object parsedExpected = RawCapableXMLResponseParser.convertRawContent((String) expected.getFieldValue(expectedFieldName));+ expected = expected.deepCopy(); // need to copy before modifying expected!+ expected.setField(expectedFieldName, parsedExpected);+ } catch (XMLStreamException ex) {+ // swallow the exception and use the un-parsed String?+ }+ }+ return super.assertRTGResults(validators, expected, actual, wt);+ }+ @Override+ public SolrParams getExtraRequestParams() {+ return extraParams;+ }public String getDefaultTransformerFactoryName() {return type;}}-++ /**+ * Local extension of XMLResponseParser that is capable of handling "raw" xml field values, for the+ * purpose of validation and consistency between expected vs. actual.+ */+ private static class RawCapableXMLResponseParser extends XMLResponseParser {++ private static String convertRawContent(String raw) throws XMLStreamException {+ return XMLResponseParser.convertRawContent(raw, (parser) -> {+ try {+ return consumeRawContent0(parser);+ } catch (XMLStreamException ex) {+ // only called in the context of this test, so the extra exception wrapping is totally fine+ throw new RuntimeException(ex);+ }+ });+ }++ protected String consumeRawContent(XMLStreamReader parser) throws XMLStreamException {+ return consumeRawContent0(parser);+ }++ private static String consumeRawContent0(XMLStreamReader parser) throws XMLStreamException {+ int depth = 0;+ StringBuilder sb = new StringBuilder();+ for (;;) {+ int elementType = parser.next();+ switch (elementType) {+ case XMLStreamConstants.START_ELEMENT:+ depth++;+ sb.append("START:").append(parser.getLocalName()).append(';');+ break;+ case XMLStreamConstants.END_ELEMENT:+ if (--depth < 0) {+ // exiting raw element+ return sb.toString();+ }+ sb.append("END:").append(parser.getLocalName()).append(';');+ break;+ case XMLStreamConstants.CHARACTERS:+ sb.append(parser.getText());+ break;+ }+ }+ }+ }/*** enforces that a valid <code>[docid]</code> is present in the response, possibly using a@@ -631,8 +876,9 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() { return USAGE.equals(resultKey) ? resultKey : resultKey+":"+USAGE; }public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {- final Object value = actual.getFirstValue(resultKey);+ final SolrDocument actual,+ final String wt) {+ Object value = normalize(wt, actual.getFirstValue(resultKey));assertNotNull(getFlParam() + " => no value in actual doc", value);assertTrue(USAGE + " must be an Integer: " + value, value instanceof Integer);@@ -664,7 +910,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() { return USAGE.equals(resultKey) ? resultKey : resultKey+":"+USAGE; }public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {final Object value = actual.getFirstValue(resultKey);assertNotNull(getFlParam() + " => no value in actual doc", value);assertTrue(USAGE + " must be an String: " + value, value instanceof String);@@ -699,8 +946,9 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() { return fl; }public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {- final Object actualVal = actual.getFirstValue(resultKey);+ final SolrDocument actual,+ final String wt) {+ Object actualVal = normalize(wt, actual.getFirstValue(resultKey));assertNotNull(getFlParam() + " => no value in actual doc", actualVal);assertEquals(getFlParam(), expectedVal, actualVal);return Collections.<String>singleton(resultKey);@@ -732,11 +980,12 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() { return fl; }public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {final Object origVal = expected.getFieldValue(fieldName);assertTrue("this validator only works on numeric fields: " + origVal, origVal instanceof Number);- assertEquals(fl, 1.3F, actual.getFirstValue(resultKey));+ assertEquals(fl, 1.3F, normalize(wt, actual.getFirstValue(resultKey)));return Collections.<String>singleton(resultKey);}}@@ -764,12 +1013,17 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public final static String SUBQ_KEY = "subq";public final static String SUBQ_FIELD = "next_2_ids_i";public String getFlParam() { return SUBQ_KEY+":["+NAME+"]"; }+ @SuppressWarnings("unchecked")public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {final int compVal = assertParseInt("expected id", expected.getFieldValue("id"));- final Object actualVal = actual.getFieldValue(SUBQ_KEY);+ Object actualVal = actual.getFieldValue(SUBQ_KEY);+ if ("json".equals(wt)) {+ actualVal = getSolrDocumentList((Map<String, Object>) actualVal);+ }assertTrue("Expected a doclist: " + actualVal,actualVal instanceof SolrDocumentList);assertTrue("should be at most 2 docs in doc list: " + actualVal,@@ -842,11 +1096,20 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() { return fl; }public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {final Object origVal = expected.getFieldValue(fieldName);assertTrue(fl + ": orig field value is not supported: " + origVal, VALUES.containsKey(origVal));-- assertEquals(fl, VALUES.get(origVal), actual.getFirstValue(resultKey));++ Object orig = VALUES.get(origVal);+ if ("json".equals(wt)) {+ try {+ orig = ObjectBuilder.fromJSON((String) orig);+ } catch (IOException ex) {+ // swallow exception and use raw `orig` String?+ }+ }+ assertEquals(fl, orig, actual.getFirstValue(resultKey));return Collections.<String>singleton(resultKey);}public Set<String> getSuppressedFields() { return Collections.singleton(fieldName); }@@ -881,7 +1144,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {final Set<String> renamed = new LinkedHashSet<>(validators.size());for (FlValidator v : validators) {@@ -895,7 +1159,7 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {for (String f : expected.getFieldNames()) {if ( matchesGlob(f) && (! renamed.contains(f) ) ) {result.add(f);- assertEquals(glob + " => " + f, expected.getFieldValue(f), actual.getFirstValue(f));+ assertEquals(glob + " => " + f, expected.getFieldValue(f), normalize(wt, actual.getFirstValue(f)));}}return result;@@ -919,7 +1183,8 @@ public class TestRandomFlRTGCloud extends SolrCloudTestCase {public String getFlParam() { return fl; }public Collection<String> assertRTGResults(final Collection<FlValidator> validators,final SolrInputDocument expected,- final SolrDocument actual) {+ final SolrDocument actual,+ final String wt) {assertEquals(fl, null, actual.getFirstValue(fieldName));return Collections.emptySet();}diff --git a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.javaindex ea0b3d3e9bf..a5a3396b368 100644--- a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java+++ b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java@@ -203,13 +203,13 @@ public class JSONWriterTest extends SolrTestCaseJ4 {methodsExpectedNotOverriden.add("writeMapOpener");methodsExpectedNotOverriden.add("writeMapSeparator");methodsExpectedNotOverriden.add("writeMapCloser");- methodsExpectedNotOverriden.add("public default void org.apache.solr.common.util.JsonTextWriter.writeArray(java.lang.String,java.util.List) throws java.io.IOException");+ methodsExpectedNotOverriden.add("public default void org.apache.solr.common.util.JsonTextWriter.writeArray(java.lang.String,java.util.List,boolean) throws java.io.IOException");methodsExpectedNotOverriden.add("writeArrayOpener");methodsExpectedNotOverriden.add("writeArraySeparator");methodsExpectedNotOverriden.add("writeArrayCloser");methodsExpectedNotOverriden.add("public default void org.apache.solr.common.util.JsonTextWriter.writeMap(org.apache.solr.common.MapWriter) throws java.io.IOException");methodsExpectedNotOverriden.add("public default void org.apache.solr.common.util.JsonTextWriter.writeIterator(org.apache.solr.common.IteratorWriter) throws java.io.IOException");- methodsExpectedNotOverriden.add("public default void org.apache.solr.common.util.JsonTextWriter.writeJsonIter(java.util.Iterator) throws java.io.IOException");+ methodsExpectedNotOverriden.add("public default void org.apache.solr.common.util.JsonTextWriter.writeJsonIter(java.util.Iterator,boolean) throws java.io.IOException");final Class<?> subClass = JSONResponseWriter.ArrayOfNameTypeValueJSONWriter.class;final Class<?> superClass = subClass.getSuperclass();diff --git a/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java b/solr/core/src/test/org/apache/solr/response/TestRawTransformer.javaindex ea5619f96a4..28fa1c2f03f 100644--- a/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java+++ b/solr/core/src/test/org/apache/solr/response/TestRawTransformer.java@@ -16,55 +16,201 @@*/package org.apache.solr.response;+import org.apache.commons.io.FileUtils;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.client.solrj.embedded.JettySolrRunner;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.NoOpResponseParser;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.SolrInputDocument;-import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.common.params.ModifiableSolrParams;import org.junit.After;+import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;+import java.io.File;+import java.lang.invoke.MethodHandles;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.Paths;+import java.util.LinkedHashMap;+import java.util.Map;+import java.util.Properties;+import java.util.regex.Pattern;+/*** Tests Raw JSON output for fields when used with and without the unique key field.** See SOLR-7993*/-public class TestRawTransformer extends SolrTestCaseJ4 {+public class TestRawTransformer extends SolrCloudTestCase {++ private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();++ /** A basic client for operations at the cloud level, default collection will be set */+ private static JettySolrRunner JSR;+ private static HttpSolrClient CLIENT;@BeforeClasspublic static void beforeClass() throws Exception {- initCore("solrconfig-doctransformers.xml", "schema.xml");+ if (random().nextBoolean()) {+ initStandalone();+ JSR.start();+ CLIENT = (HttpSolrClient) JSR.newClient();+ } else {+ initCloud();+ CLIENT = (HttpSolrClient) JSR.newClient();+ JSR = null;+ }+ initIndex();+ }++ private static void initStandalone() throws Exception {+ initCore("solrconfig-minimal.xml", "schema_latest.xml");+ File homeDir = createTempDir().toFile();+ final File collDir = new File(homeDir, "collection1");+ final File confDir = collDir.toPath().resolve("conf").toFile();+ confDir.mkdirs();+ FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(homeDir, "solr.xml"));+ String src_dir = TEST_HOME() + "/collection1/conf";+ FileUtils.copyFile(new File(src_dir, "schema_latest.xml"),+ new File(confDir, "schema.xml"));+ FileUtils.copyFile(new File(src_dir, "solrconfig-minimal.xml"),+ new File(confDir, "solrconfig.xml"));+ for (String file : new String[] {"solrconfig.snippet.randomindexconfig.xml",+ "stopwords.txt", "synonyms.txt", "protwords.txt", "currency.xml"}) {+ FileUtils.copyFile(new File(src_dir, file), new File(confDir, file));+ }+ Files.createFile(collDir.toPath().resolve("core.properties"));+ Properties nodeProperties = new Properties();+ nodeProperties.setProperty("solr.data.dir", h.getCore().getDataDir());+ JSR = new JettySolrRunner(homeDir.getAbsolutePath(), nodeProperties, buildJettyConfig("/solr"));+ }++ private static void initCloud() throws Exception {+ final String configName = DEBUG_LABEL + "_config-set";+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");++ final int numNodes = 3;+ MiniSolrCloudCluster cloud = configureCluster(numNodes).addConfig(configName, configDir).configure();++ Map<String, String> collectionProperties = new LinkedHashMap<>();+ collectionProperties.put("config", "solrconfig-minimal.xml");+ collectionProperties.put("schema", "schema_latest.xml");+ CloudSolrClient cloudSolrClient = cloud.getSolrClient();+ CollectionAdminRequest.createCollection("collection1", configName, numNodes, 1)+ .setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE)+ .setProperties(collectionProperties)+ .process(cloudSolrClient);++ JSR = cloud.getRandomJetty(random());+ }++ @AfterClass+ private static void afterClass() throws Exception{+ if (JSR != null) {+ JSR.stop();+ }+ // NOTE: CLOUD_CLIENT should be stopped automatically in `SolrCloudTestCase.shutdownCluster()`}@Afterpublic void cleanup() throws Exception {- assertU(delQ("*:*"));- assertU(commit());+ if (JSR != null) {+ assertU(delQ("*:*"));+ assertU(commit());+ }}- @Test- public void testCustomTransformer() throws Exception {+ private static final int MAX = 10;+ private static void initIndex() throws Exception {// Build a simple index- int max = 10;- for (int i = 0; i < max; i++) {+ // TODO: why are we indexing 10 docs here? Wouldn't one suffice?+ for (int i = 0; i < MAX; i++) {SolrInputDocument sdoc = new SolrInputDocument();sdoc.addField("id", i);+ // below are single-valued fieldssdoc.addField("subject", "{poffL:[{offL:[{oGUID:\"79D5A31D-B3E4-4667-B812-09DF4336B900\",oID:\"OO73XRX\",prmryO:1,oRank:1,addTp:\"Office\",addCd:\"AA4GJ5T\",ad1:\"102 S 3rd St Ste 100\",city:\"Carson City\",st:\"MI\",zip:\"48811\",lat:43.176885,lng:-84.842919,phL:[\"(989) 584-1308\"],faxL:[\"(989) 584-6453\"]}]}]}");- sdoc.addField("title", "title_" + i);- updateJ(jsonAdd(sdoc), null);+ sdoc.addField("author", "<root><child1>some</child1><child2>trivial</child2><child3>xml</child3></root>");+ // below are multiValued fields+ sdoc.addField("links", "{an_array:[1,2,3]}");+ sdoc.addField("links", "{an_array:[4,5,6]}");+ sdoc.addField("content_type", "<root>one</root>");+ sdoc.addField("content_type", "<root>two</root>");+ CLIENT.add("collection1", sdoc);}- assertU(commit());- assertQ(req("q", "*:*"), "//*[@numFound='" + max + "']");+ CLIENT.commit("collection1");+ assertEquals(MAX, CLIENT.query("collection1", new ModifiableSolrParams(Map.of("q", new String[]{"*:*"}))).getResults().getNumFound());+ }++ @Test+ public void testXmlTransformer() throws Exception {+ QueryRequest req = new QueryRequest(new ModifiableSolrParams(+ Map.of("q", new String[]{"*:*"}, "fl", new String[]{"author:[xml],content_type:[xml]"}, "wt", new String[]{"xml"})+ ));+ req.setResponseParser(XML_NOOP_RESPONSE_PARSER);+ String strResponse = (String) CLIENT.request(req,"collection1").get("response");+ assertTrue("response does not contain raw XML encoding: " + strResponse,+ strResponse.contains("<raw name=\"author\"><root><child1>some</child1><child2>trivial</child2><child3>xml</child3></root></raw>"));+ assertTrue("response (multiValued) does not contain raw XML encoding: " + strResponse,+ Pattern.compile("<arr name=\"content_type\">\\s*<raw><root>one</root></raw>\\s*<raw><root>two</root></raw>\\s*</arr>").matcher(strResponse).find());- SolrQueryRequest req = req("q", "*:*", "fl", "subject:[json]", "wt", "json");- String strResponse = h.query(req);+ req = new QueryRequest(new ModifiableSolrParams(+ Map.of("q", new String[]{"*:*"}, "fl", new String[]{"author,content_type"}, "wt", new String[]{"xml"})+ ));+ req.setResponseParser(XML_NOOP_RESPONSE_PARSER);+ strResponse = (String) CLIENT.request(req, "collection1").get("response");+ assertTrue("response does not contain escaped XML encoding: " + strResponse,+ strResponse.contains("<str name=\"author\"><root><child1"));+ assertTrue("response (multiValued) does not contain escaped XML encoding: " + strResponse,+ Pattern.compile("<arr name=\"content_type\">\\s*<str><root>").matcher(strResponse).find());++ req = new QueryRequest(new ModifiableSolrParams(+ Map.of("q", new String[]{"*:*"}, "fl", new String[]{"author:[xml],content_type:[xml]"}, "wt", new String[]{"json"})+ ));+ req.setResponseParser(JSON_NOOP_RESPONSE_PARSER);+ strResponse = (String) CLIENT.request(req, "collection1").get("response");+ assertTrue("unexpected serialization of XML field value in JSON response: " + strResponse,+ strResponse.contains("\"author\":\"<root><child1>some</child1>"));+ assertTrue("unexpected (multiValued) serialization of XML field value in JSON response: " + strResponse,+ strResponse.contains("\"content_type\":[\"<root>one</root>"));+ }++ @Test+ public void testJsonTransformer() throws Exception {+ QueryRequest req = new QueryRequest(new ModifiableSolrParams(+ Map.of("q", new String[]{"*:*"}, "fl", new String[]{"subject:[json],links:[json]"}, "wt", new String[]{"json"})+ ));+ req.setResponseParser(JSON_NOOP_RESPONSE_PARSER);+ String strResponse = (String) CLIENT.request(req,"collection1").get("response");assertTrue("response does not contain right JSON encoding: " + strResponse,- strResponse.contains("\"subject\":[{poffL:[{offL:[{oGUID:\"7"));+ strResponse.contains("\"subject\":{poffL:[{offL:[{oGUID:\"7"));+ assertTrue("response (multiValued) does not contain right JSON encoding: " + strResponse,+ Pattern.compile("\"links\":\\[\\{an_array:\\[1,2,3]},\\s*\\{an_array:\\[4,5,6]}]").matcher(strResponse).find());- req = req("q", "*:*", "fl", "id,subject", "wt", "json");- strResponse = h.query(req);+ req = new QueryRequest(new ModifiableSolrParams(+ Map.of("q", new String[]{"*:*"}, "fl", new String[]{"id", "subject,links"}, "wt", new String[]{"json"})+ ));+ req.setResponseParser(JSON_NOOP_RESPONSE_PARSER);+ strResponse = (String) CLIENT.request(req, "collection1").get("response");assertTrue("response does not contain right JSON encoding: " + strResponse,- strResponse.contains("subject\":[\""));+ strResponse.contains("subject\":\""));+ assertTrue("response (multiValued) does not contain right JSON encoding: " + strResponse,+ strResponse.contains("\"links\":[\""));}+ private static final NoOpResponseParser XML_NOOP_RESPONSE_PARSER = new NoOpResponseParser();+ private static final NoOpResponseParser JSON_NOOP_RESPONSE_PARSER = new NoOpResponseParser() {+ @Override+ public String getWriterType() {+ return "json";+ }+ };+}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.javaindex b225ceba7b9..d1e5dda4dac 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java@@ -258,7 +258,7 @@ class XLSXWriter extends TabularResponseWriter {values = tmpList;}- writeArray(xlField.name, values.iterator());+ writeArray(xlField.name, values.iterator(), false);} else {// normalize to first value@@ -278,7 +278,8 @@ class XLSXWriter extends TabularResponseWriter {}@Override- public void writeArray(String name, Iterator<?> val) throws IOException {+ public void writeArray(String name, Iterator<?> val, boolean raw) throws IOException {+ assert !raw;StringBuffer output = new StringBuffer();while (val.hasNext()) {Object v = val.next();diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex 27eb36100f4..6d0b109afa1 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -152,6 +152,13 @@ Currently this change should only effect compatibility of custom code overriding* SOLR-14510: The `writeStartDocumentList` in `TextResponseWriter` now receives an extra boolean parameter representing the "exactness" of the `numFound` value (exact vs approximation).Any custom response writer extending `TextResponseWriter` will need to implement this abstract method now (instead previous with the same name but without the new boolean parameter).+* SOLR-9376: The response format for field values serialized as raw XML (via the `[xml]` raw value DocTransformer+and `wt=xml`) has changed. Previously, values were dropped in directly as top-level child elements of each `<doc>`,+obscuring associated field names and yielding inconsistent `<doc>` structure. As of version 9.0, raw values are+wrapped in a `<raw name="field_name">[...]</raw>` element at the top level of each `<doc>` (or within an enclosing+`<arr name="field_name"><raw>[...]</raw></arr>` element for multi-valued fields). Existing clients that parse field+values serialized in this way will need to be updated accordingly.+=== solr.xml maxBooleanClauses now enforced recursivelyLucene 9.0 has additional safety checks over previous versions that impact how the `solr.xml` global `<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is enforced.diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.javaindex 40cc6499000..c219e70a9fa 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java@@ -22,12 +22,14 @@ import javax.xml.stream.XMLStreamException;import javax.xml.stream.XMLStreamReader;import java.io.InputStream;import java.io.Reader;+import java.io.StringReader;import java.lang.invoke.MethodHandles;import java.time.Instant;import java.util.ArrayList;import java.util.Date;import java.util.List;import java.util.Locale;+import java.util.function.Function;import org.apache.solr.client.solrj.ResponseParser;import org.apache.solr.common.EmptyEntityResolver;@@ -183,6 +185,7 @@ public class XMLResponseParser extends ResponseParser}},+ RAW (true) { @Override public Object read( String txt ) { return null; } },ARR (false) { @Override public Object read( String txt ) { return null; } },LST (false) { @Override public Object read( String txt ) { return null; } },RESULT (false) { @Override public Object read( String txt ) { return null; } },@@ -262,6 +265,7 @@ public class XMLResponseParser extends ResponseParsercase LONG:case NULL:case STR:+ case RAW:break;}throw new XMLStreamException( "branch element not handled!", parser.getLocation() );@@ -335,6 +339,7 @@ public class XMLResponseParser extends ResponseParsercase LONG:case NULL:case STR:+ case RAW:break;}throw new XMLStreamException( "branch element not handled!", parser.getLocation() );@@ -454,6 +459,15 @@ public class XMLResponseParser extends ResponseParser} else if( type == KnownType.LST ) {doc.addField( name, readNamedList( parser ) );depth--;+ } else if( type == KnownType.RESULT ) {+ // e.g., from the [subquery] doc transformer+ doc.put(name, readDocuments(parser));+ depth--;+ } else if( type == KnownType.RAW ) {+ // e.g., from the raw [xml] doc transformer.+ String raw = consumeRawContent(parser);+ doc.addField(name, raw);+ depth--;} else if( !type.isLeaf ) {throw new XMLStreamException( "must be value or array", parser.getLocation() );}@@ -480,5 +494,29 @@ public class XMLResponseParser extends ResponseParser}}+ /**+ * This is a stub method for handling/validating "raw" xml field values in the context of tests. Before this+ * stub method was present, "raw" content would have still thrown an exception, albeit a different, more inscrutable+ * exception.+ */+ protected String consumeRawContent(XMLStreamReader parser) throws XMLStreamException {+ throw new UnsupportedOperationException(XMLResponseParser.class + " is not capable of consuming field values serialized as raw XML");+ }++ /**+ * Convenience method that converts raw String input (should be valid xml when wrapped in a root element) and+ * converts it to a format compatible with how {@link XMLResponseParser} parses from raw xml fields.+ * This method is intended for test validation.+ *+ * The main reason this method exists is to provide a consistent way of configuring and creating and invoking+ * the sub-parser+ */+ protected static String convertRawContent(String raw, Function<XMLStreamReader, String> consumeRawContent) throws XMLStreamException {+ XMLStreamReader subParser = factory.createXMLStreamReader(new StringReader("<raw>" + raw + "</raw>"));+ while (subParser.next() != XMLStreamConstants.START_ELEMENT) {+ // consume any early stuff+ }+ return consumeRawContent.apply(subParser);+ }}diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java b/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.javaindex 42025e38786..85de57bda61 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/JsonTextWriter.java@@ -65,6 +65,10 @@ public interface JsonTextWriter extends TextWriter {_writeChar(']');}+ default void writeStrRaw(String name, String val) throws IOException {+ _writeStr(val);+ }+default void writeStr(String name, String val, boolean needsEscaping) throws IOException {// it might be more efficient to use a stringbuilder or write substrings// if writing chars to the stream is slow.@@ -185,12 +189,12 @@ public interface JsonTextWriter extends TextWriter {_writeChar(':');}- default void writeJsonIter(Iterator<?> val) throws IOException {+ default void writeJsonIter(Iterator<?> val, boolean raw) throws IOException {incLevel();boolean first = true;while (val.hasNext()) {if (!first) indent();- writeVal(null, val.next());+ writeVal(null, val.next(), raw);if (val.hasNext()) {writeArraySeparator();}@@ -264,15 +268,15 @@ public interface JsonTextWriter extends TextWriter {}- default void writeArray(String name, List<?> l) throws IOException {+ default void writeArray(String name, List<?> l, boolean raw) throws IOException {writeArrayOpener(l.size());- writeJsonIter(l.iterator());+ writeJsonIter(l.iterator(), raw);writeArrayCloser();}- default void writeArray(String name, Iterator<?> val) throws IOException {+ default void writeArray(String name, Iterator<?> val, boolean raw) throws IOException {writeArrayOpener(-1); // no trivial way to determine array size- writeJsonIter(val);+ writeJsonIter(val, raw);writeArrayCloser();}diff --git a/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java b/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.javaindex e02677fbd5b..a7605f11e6d 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/TextWriter.java@@ -43,6 +43,10 @@ import org.apache.solr.common.PushWriter;public interface TextWriter extends PushWriter {default void writeVal(String name, Object val) throws IOException {+ writeVal(name, val, false);+ }++ default void writeVal(String name, Object val, boolean raw) throws IOException {// if there get to be enough types, perhaps hashing on the type// to get a handler might be faster (but types must be exact to do that...)@@ -52,8 +56,12 @@ public interface TextWriter extends PushWriter {if (val == null) {writeNull(name);} else if (val instanceof CharSequence) {- writeStr(name, val.toString(), true);- // micro-optimization... using toString() avoids a cast first+ if (raw) {+ writeStrRaw(name, val.toString());+ } else {+ writeStr(name, val.toString(), true);+ // micro-optimization... using toString() avoids a cast first+ }} else if (val instanceof Number) {writeNumber(name, (Number) val);} else if (val instanceof Boolean) {@@ -65,9 +73,14 @@ public interface TextWriter extends PushWriter {} else if (val instanceof NamedList) {writeNamedList(name, (NamedList)val);} else if (val instanceof Path) {- writeStr(name, ((Path) val).toAbsolutePath().toString(), true);+ final String pathStr = ((Path) val).toAbsolutePath().toString();+ if (raw) {+ writeStrRaw(name, pathStr);+ } else {+ writeStr(name, pathStr, true);+ }} else if (val instanceof IteratorWriter) {- writeIterator(name, (IteratorWriter) val);+ writeIterator(name, (IteratorWriter) val, raw);} else if (val instanceof MapWriter) {writeMap(name, (MapWriter) val);} else if (val instanceof MapSerializable) {@@ -76,29 +89,39 @@ public interface TextWriter extends PushWriter {} else if (val instanceof Map) {writeMap(name, (Map)val, false, true);} else if (val instanceof Iterator) { // very generic; keep towards the end- writeArray(name, (Iterator) val);+ writeArray(name, (Iterator) val, raw);} else if (val instanceof Iterable) { // very generic; keep towards the end- writeArray(name,((Iterable)val).iterator());+ writeArray(name,((Iterable)val).iterator(), raw);} else if (val instanceof Object[]) {- writeArray(name,(Object[])val);+ writeArray(name,(Object[])val, raw);} else if (val instanceof byte[]) {byte[] arr = (byte[])val;writeByteArr(name, arr, 0, arr.length);} else if (val instanceof EnumFieldValue) {- writeStr(name, val.toString(), true);- } else if (val instanceof WriteableValue) {- ((WriteableValue)val).write(name, this);+ if (raw) {+ writeStrRaw(name, val.toString());+ } else {+ writeStr(name, val.toString(), true);+ }} else {// default... for debugging only. Would be nice to "assert false" ?writeStr(name, val.getClass().getName() + ':' + val.toString(), true);}}+ /**+ * Writes the specified val directly to the backing writer, without wrapping (e.g., in quotes) or escaping+ * of any kind.+ */+ default void writeStrRaw(String name, String val) throws IOException {+ throw new UnsupportedOperationException();+ }+void writeStr(String name, String val, boolean needsEscaping) throws IOException;void writeMap(String name, Map<?, ?> val, boolean excludeOuter, boolean isFirstVal) throws IOException;- void writeArray(String name, Iterator<?> val) throws IOException;+ void writeArray(String name, Iterator<?> val, boolean raw) throws IOException;void writeNull(String name) throws IOException;@@ -153,12 +176,12 @@ public interface TextWriter extends PushWriter {}}- default void writeArray(String name, Object[] val) throws IOException {- writeArray(name, Arrays.asList(val));+ default void writeArray(String name, Object[] val, boolean raw) throws IOException {+ writeArray(name, Arrays.asList(val), raw);}- default void writeArray(String name, List<?> l) throws IOException {- writeArray(name, l.iterator());+ default void writeArray(String name, List<?> l, boolean raw) throws IOException {+ writeArray(name, l.iterator(), raw);}@@ -224,7 +247,7 @@ public interface TextWriter extends PushWriter {/*todo*/}- default void writeIterator(String name, IteratorWriter iw) throws IOException {+ default void writeIterator(String name, IteratorWriter iw, boolean raw) throws IOException {writeIterator(iw);}diff --git a/solr/solrj/src/java/org/apache/solr/common/util/WriteableValue.java b/solr/solrj/src/java/org/apache/solr/common/util/WriteableValue.javadeleted file mode 100644index 82e7d00e0fc..00000000000--- a/solr/solrj/src/java/org/apache/solr/common/util/WriteableValue.java+++ /dev/null@@ -1,25 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.common.util;--import java.io.IOException;--import org.apache.solr.common.util.JavaBinCodec.ObjectResolver;--public abstract class WriteableValue implements ObjectResolver {- public abstract void write(String name, TextWriter writer) throws IOException;-}\ No newline at end of file