Apache Solr open-source search software
SOLR-10198: EmbeddedSolrServer StreamingResponseCallback: to SolrJ-native types. (#4965)
Nicke97c2a0884cb
diff --git a/changelog/unreleased/SOLR-10198.yml b/changelog/unreleased/SOLR-10198.ymlnew file mode 100644index 00000000000..f10ba33fee6--- /dev/null+++ b/changelog/unreleased/SOLR-10198.yml@@ -0,0 +1,7 @@+title: "EmbeddedSolrServer StreamingResponseCallback: Normalize Lucene stored fields to SolrJ-native types."+type: fixed+authors:+ - name: Nick Shanin+links:+ - name: SOLR-10198+ url: https://issues.apache.org/jira/browse/SOLR-10198diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.javaindex 222b01d7957..2c280bf8bfc 100644--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java@@ -54,9 +54,11 @@ import org.apache.solr.core.SolrCore;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.request.SolrRequestHandler;import org.apache.solr.request.SolrRequestInfo;+import org.apache.solr.response.DocsStreamer;import org.apache.solr.response.JavaBinResponseWriter;import org.apache.solr.response.ResultContext;import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.schema.IndexSchema;import org.apache.solr.servlet.SolrRequestParsers;/**@@ -290,7 +292,7 @@ public class EmbeddedSolrServer extends SolrClient {};// invoke callbacks, and writes the rest to byteBuffer- try (var javaBinCodec = createJavaBinCodec(callback, resolver)) {+ try (var javaBinCodec = createJavaBinCodec(callback, resolver, req.getSchema())) {javaBinCodec.setWritableDocFields(resolver).marshal(rsp.getValues(), byteBuffer);}}@@ -372,12 +374,17 @@ public class EmbeddedSolrServer extends SolrClient {}private JavaBinCodec createJavaBinCodec(- final StreamingResponseCallback callback, final JavaBinResponseWriter.Resolver resolver) {+ final StreamingResponseCallback callback,+ final JavaBinResponseWriter.Resolver resolver,+ final IndexSchema schema) {return new JavaBinCodec(resolver) {@Overridepublic void writeSolrDocument(SolrDocument doc) {- callback.streamSolrDocument(doc);+ // The callback receives the document itself, so it bypasses the conversion the+ // Resolver applies when a document is serialized. Convert stored fields here so+ // streamed documents carry the same native types as query() results.+ callback.streamSolrDocument(DocsStreamer.externalizeStoredValues(doc, schema));// super.writeSolrDocument( doc, fields );}diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.javaindex 1f7a2e5ac28..7b33cf5800c 100644--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java@@ -19,10 +19,14 @@ package org.apache.solr.response;import static org.apache.solr.schema.FieldType.ExternalizeStoredValuesAsObjects;import java.io.IOException;+import java.lang.invoke.MethodHandles;import java.util.ArrayList;+import java.util.Collection;import java.util.HashSet;import java.util.Iterator;import java.util.List;+import java.util.Map;+import java.util.Objects;import java.util.Set;import org.apache.lucene.document.Document;import org.apache.lucene.index.IndexableField;@@ -54,9 +58,14 @@ import org.apache.solr.search.DocList;import org.apache.solr.search.ReturnFields;import org.apache.solr.search.SolrDocumentFetcher;import org.apache.solr.search.SolrReturnFields;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;/** This streams SolrDocuments from a DocList and applies transformer */public class DocsStreamer implements Iterator<SolrDocument> {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ private static final Object FAILED_STORED_VALUE = new Object();+/*** A hardcoded list of known Solr field types that will be trusted to control their own conversion* of stored field values into external Objects (via {@link FieldType#toObject}) when returning@@ -186,6 +195,86 @@ public class DocsStreamer implements Iterator<SolrDocument> {public void remove() { // do nothing}+ /**+ * Replace Lucene {@link IndexableField} values on a {@link SolrDocument} (and nested / child+ * documents) with the SolrJ-native objects that clients see after JavaBin deserialization. The+ * serialized JavaBin path already converts stored values through JavaBinResponseWriter.Resolver;+ * this exists for the EmbeddedSolrServer streaming path, whose codec hands documents straight to+ * the callback, so {@code queryAndStreamResponse} matches {@code query} / {@code HttpSolrClient}.+ *+ * <p>A stored value that cannot be converted is logged and omitted; conversion continues for the+ * remaining values and fields.+ *+ * <p>Do not call this from {@link #convertLuceneDocToSolrDoc}; JSON/XML writers and some+ * transformers still expect stored fields as {@link IndexableField}.+ *+ * @see #getValue(SchemaField, IndexableField)+ */+ public static SolrDocument externalizeStoredValues(SolrDocument doc, IndexSchema schema) {+ if (doc == null || schema == null) {+ return doc;+ }+ List<String> failedFields = null;+ for (Iterator<Map.Entry<String, Object>> it = doc.iterator(); it.hasNext(); ) {+ Map.Entry<String, Object> entry = it.next();+ Object val = entry.getValue();+ Object converted = externalizeValue(val, schema);+ if (FAILED_STORED_VALUE.equals(converted)) {+ // The document's entry iterator does not support remove(), so collect the+ // failed fields and remove them from the document after the loop instead.+ if (failedFields == null) {+ failedFields = new ArrayList<>();+ }+ failedFields.add(entry.getKey());+ } else if (!Objects.equals(converted, val)) {+ entry.setValue(converted);+ }+ }+ if (failedFields != null) {+ for (String failedField : failedFields) {+ doc.remove(failedField);+ }+ }+ List<SolrDocument> children = doc.getChildDocuments();+ if (children != null) {+ for (SolrDocument child : children) {+ externalizeStoredValues(child, schema);+ }+ }+ return doc;+ }++ private static Object externalizeValue(Object val, IndexSchema schema) {+ if (val instanceof IndexableField f) {+ try {+ return getValue(schema.getFieldOrNull(f.name()), f);+ } catch (Exception | AssertionError e) {+ // AssertionError: point field types (IntPointField, DatePointField, ...) throw it+ // from toObject when a stored value predates a change to a point type.+ log.warn("Error reading a field : {}", f, e);+ return FAILED_STORED_VALUE;+ }+ }+ if (val instanceof SolrDocument nested) {+ return externalizeStoredValues(nested, schema);+ }+ if (val instanceof Collection<?> coll) {+ List<Object> out = new ArrayList<>(coll.size());+ boolean changed = false;+ for (Object item : coll) {+ Object converted = externalizeValue(item, schema);+ if (FAILED_STORED_VALUE.equals(converted)) {+ changed = true;+ continue;+ }+ changed |= !Objects.equals(converted, item);+ out.add(converted);+ }+ return changed ? (out.isEmpty() ? FAILED_STORED_VALUE : out) : val;+ }+ return val;+ }+public static Object getValue(SchemaField sf, IndexableField f) {FieldType ft = null;if (sf != null) {diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerStreamingTypes.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerStreamingTypes.javanew file mode 100644index 00000000000..a77d1cd01fd--- /dev/null+++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerStreamingTypes.java@@ -0,0 +1,186 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.solrj.embedded;++import java.util.Arrays;+import java.util.List;+import java.util.concurrent.atomic.AtomicReference;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.StoredField;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexableField;+import org.apache.solr.SolrTestCase;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.client.solrj.response.StreamingResponseCallback;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.core.SolrCore;+import org.apache.solr.schema.IndexSchema;+import org.apache.solr.util.EmbeddedSolrServerTestRule;+import org.apache.solr.util.RefCounted;+import org.junit.BeforeClass;+import org.junit.ClassRule;+import org.junit.Test;++/**+ * EmbeddedSolrServer {@code queryAndStreamResponse} must return the same native field types as+ * {@code query} / HttpSolrClient, not Lucene {@link IndexableField} instances.+ */+public class TestEmbeddedSolrServerStreamingTypes extends SolrTestCase {++ @ClassRule+ public static final EmbeddedSolrServerTestRule solrTestRule = new EmbeddedSolrServerTestRule();++ @BeforeClass+ public static void beforeClass() throws Exception {+ solrTestRule.startSolr(SolrTestCaseJ4.TEST_HOME());+ SolrTestCaseJ4.newRandomConfig();+ solrTestRule.newCollection().withConfigSet(SolrTestCaseJ4.TEST_COLL1_CONF()).create();+ }++ @Test+ public void testQueryAndStreamResponseReturnsNativeFieldTypes() throws Exception {+ EmbeddedSolrServer server = solrTestRule.getSolrClient("collection1");+ server.deleteByQuery("*:*");+ // city_s1 is *_s1: stored string, multiValued=false. schema.xml is version 1.0, so+ // fields without multiValued="false" (including name) default to multiValued and+ // query() returns a one-element list. foo_i_p is pint so the native type is Integer.+ // name is multiValued: streaming must convert each element (Collection branch).+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", "1");+ doc.addField("foo_i_p", 42);+ doc.addField("city_s1", "Boston");+ doc.addField("name", "Alice");+ doc.addField("name", "Bob");+ server.add(doc);+ server.commit();++ SolrQuery q = new SolrQuery("*:*");+ q.setFields("id", "foo_i_p", "city_s1", "name");++ QueryResponse rsp = server.query(q);+ SolrDocument queried = rsp.getResults().get(0);+ Object queriedId = queried.getFieldValue("id");+ Object queriedInt = queried.getFieldValue("foo_i_p");+ Object queriedName = queried.getFieldValue("city_s1");+ assertEquals("1", queriedId);+ assertEquals(Integer.valueOf(42), queriedInt);+ assertEquals("Boston", queriedName);+ assertFalse("query() must not leak Lucene stored fields", queriedInt instanceof IndexableField);++ AtomicReference<SolrDocument> streamed = new AtomicReference<>();+ server.queryAndStreamResponse(+ q,+ new StreamingResponseCallback() {+ @Override+ public void streamSolrDocument(SolrDocument doc) {+ streamed.set(doc);+ }++ @Override+ public void streamDocListInfo(long numFound, long start, Float maxScore) {+ assertEquals(1, numFound);+ }+ });++ SolrDocument streamedDoc = streamed.get();+ assertNotNull(streamedDoc);+ Object streamedId = streamedDoc.getFieldValue("id");+ Object streamedInt = streamedDoc.getFieldValue("foo_i_p");+ Object streamedName = streamedDoc.getFieldValue("city_s1");+ assertFalse(+ "queryAndStreamResponse must not leak Lucene stored fields",+ streamedInt instanceof IndexableField);+ assertEquals(queriedId.getClass(), streamedId.getClass());+ assertEquals(queriedInt.getClass(), streamedInt.getClass());+ assertEquals(queriedName.getClass(), streamedName.getClass());+ assertEquals(queriedId, streamedId);+ assertEquals(queriedInt, streamedInt);+ assertEquals(queriedName, streamedName);+ assertEquals(Integer.class, streamedInt.getClass());+ assertEquals(String.class, streamedName.getClass());+ // multi-valued field: both paths return a list of native Strings+ Object queriedNames = queried.getFieldValue("name");+ Object streamedNames = streamedDoc.getFieldValue("name");+ assertTrue(queriedNames instanceof List);+ assertTrue(streamedNames instanceof List);+ assertEquals(queriedNames, streamedNames);+ assertEquals(Arrays.asList("Alice", "Bob"), streamedNames);+ }++ @Test+ public void testStreamingOmitsValueWhoseConversionThrowsAssertionError() throws Exception {+ EmbeddedSolrServer server = solrTestRule.getSolrClient("collection1");+ server.deleteByQuery("*:*");+ server.commit();++ // Write a Lucene document directly, bypassing the update pipeline: count_i_p is a+ // pint field now, but the stored value written here is a string, as if it had been+ // indexed before the field's type changed to pint. (A pint field the other test in+ // this class does not use, so the raw field cannot clash with its field metadata.)+ // IntPointField.toObject throws AssertionError for such a value (it has no numeric+ // value); that must not fail the whole streamed response. The value is logged and+ // omitted instead. The other fields are created through the schema so their Lucene+ // field shapes match documents the update pipeline writes.+ Document luceneDoc = new Document();+ luceneDoc.add(new StoredField("count_i_p", "not-a-number"));+ try (SolrCore core = solrTestRule.getCoreContainer().getCore("collection1")) {+ IndexSchema schema = core.getLatestSchema();+ for (IndexableField f : schema.getField("id").createFields("bad1")) {+ luceneDoc.add(f);+ }+ for (IndexableField f : schema.getField("city_s1").createFields("Boston")) {+ luceneDoc.add(f);+ }+ RefCounted<IndexWriter> iwRef = core.getSolrCoreState().getIndexWriter(core);+ try {+ iwRef.get().addDocument(luceneDoc);+ } finally {+ iwRef.decref();+ }+ }+ server.commit();++ SolrQuery q = new SolrQuery("*:*");+ q.setFields("id", "count_i_p", "city_s1");++ AtomicReference<SolrDocument> streamed = new AtomicReference<>();+ server.queryAndStreamResponse(+ q,+ new StreamingResponseCallback() {+ @Override+ public void streamSolrDocument(SolrDocument doc) {+ streamed.set(doc);+ }++ @Override+ public void streamDocListInfo(long numFound, long start, Float maxScore) {+ assertEquals(1, numFound);+ }+ });++ SolrDocument streamedDoc = streamed.get();+ assertNotNull("the document must still be streamed", streamedDoc);+ assertEquals("bad1", streamedDoc.getFieldValue("id"));+ assertEquals("Boston", streamedDoc.getFieldValue("city_s1"));+ assertFalse(+ "the unconvertible value must be omitted, not streamed",+ streamedDoc.containsKey("count_i_p"));+ }+}diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java b/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.javaindex 524b38100ae..15af65de82f 100644--- a/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java+++ b/solr/core/src/test/org/apache/solr/handler/tagger/EmbeddedSolrNoSerializeTest.java@@ -25,7 +25,6 @@ package org.apache.solr.handler.tagger;import java.io.IOException;import java.util.concurrent.atomic.AtomicReference;import java.util.function.BiFunction;-import org.apache.lucene.document.Field;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;@@ -130,6 +129,6 @@ public class EmbeddedSolrNoSerializeTest extends SolrTestCaseJ4 {QueryResponse rsp = req.process(solrServer);assertNotNull(rsp.getResponse().get("tags"));assertNotNull(refDoc.get());- assertEquals("Boston", ((Field) refDoc.get().getFieldValue("name")).stringValue());+ assertEquals("Boston", refDoc.get().getFieldValue("name"));}}