Apache Solr open-source search software
SOLR-16589: Large fields with large=true can be truncated when using unicode values (#1241)
Kevin Risden45834aa62454
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 3c2b73e1a8d..18624b98cad 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -130,6 +130,8 @@ Bug Fixes* SOLR-10458: Fix followRedirect property on HttpSolrClient not set when using Builder pattern. (Eric Pugh)+* SOLR-16589: Large fields with large=true can be truncated when using unicode values (Kevin Risden)+Build---------------------* Upgrade forbiddenapis to 3.4 (Uwe Schindler)diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.javaindex 6eedcb06160..ca231379cb8 100644--- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java+++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java@@ -508,7 +508,7 @@ public class SolrDocumentFetcher {public void stringField(FieldInfo fieldInfo, String value) throws IOException {Objects.requireNonNull(value, "String value should not be null");bytesRef.bytes = value.getBytes(StandardCharsets.UTF_8);- bytesRef.length = value.length();+ bytesRef.length = bytesRef.bytes.length;done = true;}diff --git a/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java b/solr/core/src/test/org/apache/solr/search/LargeFieldTest.javaindex d85be338a50..5045a3561bd 100644--- a/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java+++ b/solr/core/src/test/org/apache/solr/search/LargeFieldTest.java@@ -23,6 +23,7 @@ import java.util.Objects;import org.apache.lucene.document.Document;import org.apache.lucene.index.IndexableField;import org.apache.lucene.misc.document.LazyDocument;+import org.apache.lucene.tests.util.TestUtil;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.schema.IndexSchema;import org.junit.AfterClass;@@ -36,7 +37,6 @@ public class LargeFieldTest extends SolrTestCaseJ4 {private static final String BIG_FIELD = "bigField";@BeforeClass- @SuppressWarnings({"unchecked"})public static void initManagedSchemaCore() throws Exception {// This testing approach means no schema file or per-test temp solr-home!System.setProperty("managed.schema.mutable", "true");@@ -85,7 +85,8 @@ public class LargeFieldTest extends SolrTestCaseJ4 {@Testpublic void test() throws Exception {// add just one document (docid 0)- assertU(adoc(ID_FLD, "101", LAZY_FIELD, "lzy", BIG_FIELD, "big document field one"));+ String bigFieldValue = TestUtil.randomUnicodeString(random());+ assertU(adoc(ID_FLD, "101", LAZY_FIELD, "lzy", BIG_FIELD, bigFieldValue));assertU(commit());// trigger the ID_FLD to get into the doc cache; don't reference other fields@@ -109,6 +110,8 @@ public class LargeFieldTest extends SolrTestCaseJ4 {assertEager(d, ID_FLD);assertLazyLoaded(d, LAZY_FIELD);assertLazyLoaded(d, BIG_FIELD); // loaded now++ assertEquals(bigFieldValue, d.getField(BIG_FIELD).stringValue());}private void assertEager(Document d, String fieldName) {