Apache Solr open-source search software
SOLR-14971: Handle atomic-removes on uncommitted docs (#2056)
Jason Gerlowskia7197ac0ce83
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 69e4b633abb..96a721af002 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -192,6 +192,8 @@ Bug Fixes* SOLR-14961: Fix for deleting zookeeper nodes with same path length. Only the first zk-node was removed. (Michael Aleythe via Mike Drob)+* SOLR-14971: AtomicUpdate 'remove', 'add-distinct' operations now works on numeric, date fields in uncommitted docs (Jason Gerlowski)+Other Changes---------------------diff --git a/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java b/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.javaindex 673c4fae191..bdc4a72aaf8 100644--- a/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java+++ b/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java@@ -16,21 +16,6 @@*/package org.apache.solr.update.processor;-import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.util.ArrayList;-import java.util.Collection;-import java.util.Collections;-import java.util.HashSet;-import java.util.LinkedHashSet;-import java.util.List;-import java.util.Map;-import java.util.Map.Entry;-import java.util.Set;-import java.util.regex.Matcher;-import java.util.regex.Pattern;-import java.util.stream.Stream;-import org.apache.commons.lang3.tuple.Pair;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.util.BytesRef;@@ -53,10 +38,20 @@ import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.NumericValueFieldType;import org.apache.solr.schema.SchemaField;import org.apache.solr.update.AddUpdateCommand;+import org.apache.solr.util.DateMathParser;import org.apache.solr.util.RefCounted;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.util.*;+import java.util.Map.Entry;+import java.util.function.BiConsumer;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import java.util.stream.Stream;+import static org.apache.solr.common.params.CommonParams.ID;/**@@ -189,7 +184,7 @@ public class AtomicUpdateDocumentMerger {if (fieldName.equals(uniqueKeyFieldName)|| fieldName.equals(CommonParams.VERSION_FIELD)|| fieldName.equals(routeFieldOrNull)) {- if (fieldValue instanceof Map ) {+ if (fieldValue instanceof Map) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Updating unique key, version or route field is not allowed: " + sdoc.getField(fieldName));} else {@@ -450,9 +445,6 @@ public class AtomicUpdateDocumentMerger {final String name = sif.getName();SolrInputField existingField = toDoc.get(name);- // throws exception if field doesn't exist- SchemaField sf = schema.getField(name);-Collection<Object> original = existingField != null ?existingField.getValues() :new ArrayList<>();@@ -460,16 +452,10 @@ public class AtomicUpdateDocumentMerger {int initialSize = original.size();if (fieldVal instanceof Collection) {for (Object object : (Collection) fieldVal) {- Object obj = sf.getType().toNativeType(object);- if (!original.contains(obj)) {- original.add(obj);- }+ addValueIfDistinct(name, original, object);}} else {- Object object = sf.getType().toNativeType(fieldVal);- if (!original.contains(object)) {- original.add(object);- }+ addValueIfDistinct(name, original, fieldVal);}if (original.size() > initialSize) { // update only if more are added@@ -516,7 +502,7 @@ public class AtomicUpdateDocumentMerger {SolrInputField existingField = toDoc.get(name);if (existingField == null) return;@SuppressWarnings({"rawtypes"})- final Collection original = existingField.getValues();+ final Collection<Object> original = existingField.getValues();if (fieldVal instanceof Collection) {for (Object object : (Collection) fieldVal) {removeObj(original, object, name);@@ -582,11 +568,11 @@ public class AtomicUpdateDocumentMerger {return objValues.iterator().next() instanceof SolrDocumentBase;}- private void removeObj(@SuppressWarnings({"rawtypes"})Collection original, Object toRemove, String fieldName) {+ private void removeObj(Collection<Object> original, Object toRemove, String fieldName) {if(isChildDoc(toRemove)) {removeChildDoc(original, (SolrInputDocument) toRemove);} else {- original.remove(getNativeFieldValue(fieldName, toRemove));+ removeFieldValueWithNumericFudging(fieldName, original, toRemove);}}@@ -600,6 +586,81 @@ public class AtomicUpdateDocumentMerger {}}+ private void removeFieldValueWithNumericFudging(String fieldName, @SuppressWarnings({"rawtypes"}) Collection<Object> original, Object toRemove) {+ if (original.size() == 0) {+ return;+ }++ final BiConsumer<Collection<Object>, Object> removePredicate = (coll, existingElement) -> coll.remove(existingElement);+ modifyCollectionBasedOnFuzzyPresence(fieldName, original, toRemove, removePredicate, null);+ }++ private void addValueIfDistinct(String fieldName, Collection<Object> original, Object toAdd) {+ final BiConsumer<Collection<Object>, Object> addPredicate = (coll, newElement) -> coll.add(newElement);+ modifyCollectionBasedOnFuzzyPresence(fieldName, original, toAdd, null, addPredicate);+ }++ /**+ * Modifies a collection based on the (loosely-judged) presence or absence of a specific value+ *+ * Several classes of atomic update (notably 'remove' and 'add-distinct') rely on being able to identify whether an+ * item is already present in a given list of values. Unfortunately the 'item' being checked for may be of different+ * types based on the format of the user request and on where the existing document was pulled from (tlog vs index).+ * As a result atomic updates needs a "fuzzy" way of checking presence and equality that is more flexible than+ * traditional equality checks allow. This method does light type-checking to catch some of these more common cases+ * (Long compared against Integers, String compared against Date, etc.), and calls the provided lambda to modify the+ * field values as necessary.+ *+ * @param fieldName the field name involved in this atomic update operation+ * @param original the list of values currently present in the existing document+ * @param rawValue a value to be checked for in 'original'+ * @param ifPresent a function to execute if rawValue was found in 'original'+ * @param ifAbsent a function to execute if rawValue was not found in 'original'+ */+ private void modifyCollectionBasedOnFuzzyPresence(String fieldName, Collection<Object> original, Object rawValue,+ BiConsumer<Collection<Object>, Object> ifPresent,+ BiConsumer<Collection<Object>, Object> ifAbsent) {+ Object nativeValue = getNativeFieldValue(fieldName, rawValue);+ Optional<Object> matchingValue = findObjectWithTypeFuzziness(original, rawValue, nativeValue);+ if (matchingValue.isPresent() && ifPresent != null) {+ ifPresent.accept(original, matchingValue.get());+ } else if(matchingValue.isEmpty() && ifAbsent != null) {+ ifAbsent.accept(original, rawValue);+ }+ }++ private Optional<Object> findObjectWithTypeFuzziness(Collection<Object> original, Object rawValue, Object nativeValue) {+ if (nativeValue instanceof Double || nativeValue instanceof Float) {+ final Number nativeAsNumber = (Number) nativeValue;+ return original.stream().filter(val ->+ val.equals(rawValue) ||+ val.equals(nativeValue) ||+ (val instanceof Number && ((Number) val).doubleValue() == nativeAsNumber.doubleValue()) ||+ (val instanceof String && val.equals(nativeAsNumber.toString())))+ .findFirst();+ } else if (nativeValue instanceof Long || nativeValue instanceof Integer) {+ final Number nativeAsNumber = (Number) nativeValue;+ return original.stream().filter(val ->+ val.equals(rawValue) ||+ val.equals(nativeValue) ||+ (val instanceof Number && ((Number) val).longValue() == nativeAsNumber.longValue()) ||+ (val instanceof String && val.equals(nativeAsNumber.toString())))+ .findFirst();+ } else if (nativeValue instanceof Date) {+ return original.stream().filter(val ->+ val.equals(rawValue) ||+ val.equals(nativeValue) ||+ (val instanceof String && DateMathParser.parseMath(null, (String)val).equals(nativeValue)))+ .findFirst();+ } else if (original.contains(nativeValue)) {+ return Optional.of(nativeValue);+ } else if (original.contains(rawValue)) {+ return Optional.of(rawValue);+ } else {+ return Optional.empty();+ }+ }+/**** @param doc document to search fordiff --git a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdateJavabinTest.java b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdateJavabinTest.javanew file mode 100644index 00000000000..758de5f37ab--- /dev/null+++ b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdateJavabinTest.java@@ -0,0 +1,370 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.update.processor;++import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.Test;++import java.time.Instant;+import java.util.Collection;+import java.util.Date;+import java.util.HashMap;+import java.util.Map;++/**+ * Tests Solr's atomic-update functionality using requests sent through SolrJ using wt=javabin+ *+ * {@link AtomicUpdatesTest} covers some of the same functionality, but does so by making xml-based requests. Recent+ * changes to Solr have made it possible for the same data sent with different formats to result in different NamedLists+ * after unmarshalling, so the test duplication is now necessary. See SOLR-13331 for an example.+ */+public class AtomicUpdateJavabinTest extends SolrCloudTestCase {+ private static final String COMMITTED_DOC_ID = "1";+ private static final String COMMITTED_DOC_STR_VALUES_ID = "1s";+ private static final String UNCOMMITTED_DOC_ID = "2";+ private static final String UNCOMMITTED_DOC_STR_VALUES_ID = "2s";+ private static final String COLLECTION = "collection1";+ private static final int NUM_SHARDS = 1;+ private static final int NUM_REPLICAS = 1;+ private static final Date DATE_1 = Date.from(Instant.ofEpochSecond(1554243309));+ private static final String DATE_1_STR = "2019-04-02T22:15:09Z";+ private static final Date DATE_2 = Date.from(Instant.ofEpochSecond(1554243609));+ private static final String DATE_2_STR = "2019-04-02T22:20:09Z";+ private static final Date DATE_3 = Date.from(Instant.ofEpochSecond(1554243909));+ private static final String DATE_3_STR = "2019-04-02T22:25:09Z";+++ @BeforeClass+ public static void setupCluster() throws Exception {+ configureCluster(1)+ .addConfig("conf", configset("cloud-dynamic"))+ .configure();++ CollectionAdminRequest.createCollection(COLLECTION, "conf", NUM_SHARDS, NUM_REPLICAS)+ .process(cluster.getSolrClient());++ cluster.waitForActiveCollection(COLLECTION, 1, 1);+ }++ @Before+ public void setUp() throws Exception {+ super.setUp();++ final SolrInputDocument committedDoc = sdoc(+ "id", COMMITTED_DOC_ID,+ "title_s", "title_1", "title_s", "title_2",+ "tv_mv_text", "text_1", "tv_mv_text", "text_2",+ "count_is", 1, "count_is", 2,+ "count_md", 1.0, "count_md", 2.0,+ "timestamps_mdt", DATE_1, "timestamps_mdt", DATE_2);+ final SolrInputDocument committedStrDoc = sdoc(+ "id", COMMITTED_DOC_STR_VALUES_ID,+ "title_s", "title_1", "title_s", "title_2",+ "tv_mv_text", "text_1", "tv_mv_text", "text_2",+ "count_is", "1", "count_is", "2",+ "count_md", "1.0", "count_md", "2.0",+ "timestamps_mdt", DATE_1_STR, "timestamps_mdt", DATE_2_STR);+ final UpdateRequest committedRequest = new UpdateRequest()+ .add(committedDoc)+ .add(committedStrDoc);+ committedRequest.commit(cluster.getSolrClient(), COLLECTION);++ // Upload a copy of id:1 that's uncommitted to test how atomic-updates modify values in the tlog+ // See SOLR-14971 for an example of why this case needs tested separately+ final SolrInputDocument uncommittedDoc = sdoc(+ "id", UNCOMMITTED_DOC_ID,+ "title_s", "title_1", "title_s", "title_2",+ "tv_mv_text", "text_1", "tv_mv_text", "text_2",+ "count_is", 1, "count_is", 2,+ "count_md", 1.0, "count_md", 2.0,+ "timestamps_mdt", DATE_1, "timestamps_mdt", DATE_2);+ final SolrInputDocument uncommittedStrDoc = sdoc(+ "id", UNCOMMITTED_DOC_STR_VALUES_ID,+ "title_s", "title_1", "title_s", "title_2",+ "tv_mv_text", "text_1", "tv_mv_text", "text_2",+ "count_is", "1", "count_is", "2",+ "count_md", "1.0", "count_md", "2.0",+ "timestamps_mdt", DATE_1_STR, "timestamps_mdt", DATE_2_STR);+ final UpdateRequest uncommittedRequest = new UpdateRequest()+ .add(uncommittedDoc)+ .add(uncommittedStrDoc);+ uncommittedRequest.process(cluster.getSolrClient(), COLLECTION);+ }++ @Test+ public void testAtomicUpdateRemovalOfStrField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ atomicRemoveValueFromField(COMMITTED_DOC_ID, "title_s", "title_1");+ ensureFieldHasValues(COMMITTED_DOC_ID, "title_s", "title_2");++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ atomicRemoveValueFromField(UNCOMMITTED_DOC_ID, "title_s", "title_1");+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "title_s", "title_2");+ }++ @Test+ public void testAtomicUpdateRemovalOfTextField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ atomicRemoveValueFromField(COMMITTED_DOC_ID, "tv_mv_text", "text_1");+ ensureFieldHasValues(COMMITTED_DOC_ID, "tv_mv_text", "text_2");++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ atomicRemoveValueFromField(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_1");+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_2");+ }++ @Test+ public void testAtomicUpdateRemovalOfIntField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_is", 1, 2);+ atomicRemoveValueFromField(COMMITTED_DOC_ID, "count_is", 1);+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_is", 2);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_is", 1, 2);+ atomicRemoveValueFromField(UNCOMMITTED_DOC_ID, "count_is", 1);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_is", 2);+ }++ @Test+ public void testAtomicUpdateRemovalOfDoubleField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_md", 1.0, 2.0);+ atomicRemoveValueFromField(COMMITTED_DOC_ID, "count_md", 1.0);+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_md", 2.0);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_md", 1.0, 2.0);+ atomicRemoveValueFromField(UNCOMMITTED_DOC_ID, "count_md", 1.0);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_md", 2.0);+ }++ @Test+ public void testAtomicUpdateRemovalOfDateField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicRemoveValueFromField(COMMITTED_DOC_ID, "timestamps_mdt", DATE_1);+ ensureFieldHasValues(COMMITTED_DOC_ID, "timestamps_mdt", DATE_2);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicRemoveValueFromField(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_1);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_2);+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDistinctValueOnStrField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "title_s", "title_3");+ ensureFieldHasValues(COMMITTED_DOC_ID, "title_s", "title_1", "title_2", "title_3");++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "title_s", "title_3");+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "title_s", "title_1", "title_2", "title_3");+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDuplicateValueOnStrField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "title_s", "title_2");+ ensureFieldHasValues(COMMITTED_DOC_ID, "title_s", "title_1", "title_2");++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "title_s", "title_2");+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "title_s", "title_1", "title_2");+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDistinctValueOnTextField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "tv_mv_text", "text_3");+ ensureFieldHasValues(COMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2", "text_3");++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_3");+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2", "text_3");+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDuplicateValueOnTextField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "tv_mv_text", "text_2");+ ensureFieldHasValues(COMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_2");+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "tv_mv_text", "text_1", "text_2");+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDistinctValueOnIntField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "count_is", 3);+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_is", 1, 2, 3);++ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(COMMITTED_DOC_STR_VALUES_ID, "count_is", 3);+ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2, 3);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "count_is", 3);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_is", 1, 2, 3);++ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_STR_VALUES_ID, "count_is", 3);+ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2, 3);+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDuplicateValueOnIntField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "count_is", 2);+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_is", 1, 2);++ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(COMMITTED_DOC_STR_VALUES_ID, "count_is", 2);+ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "count_is", 2);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_is", 1, 2);++ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_STR_VALUES_ID, "count_is", 2);+ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_is", 1, 2);+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDistinctValueOnDoubleField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "count_md", 3.0);+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_md", 1.0, 2.0, 3.0);++ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(COMMITTED_DOC_STR_VALUES_ID, "count_md", 3.0);+ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0, 3.0);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "count_md", 3.0);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_md", 1.0, 2.0, 3.0);++ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_STR_VALUES_ID, "count_md", 3.0);+ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0, 3.0);+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDuplicateValueOnDoubleField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "count_md", 2.0);+ ensureFieldHasValues(COMMITTED_DOC_ID, "count_md", 1.0, 2.0);++ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(COMMITTED_DOC_STR_VALUES_ID, "count_md", 2.0);+ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "count_md", 2.0);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "count_md", 1.0, 2.0);++ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_STR_VALUES_ID, "count_md", 2.0);+ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "count_md", 1.0, 2.0);+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDistinctValueOnDateField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "timestamps_mdt", DATE_3);+ ensureFieldHasValues(COMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2, DATE_3);++ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(COMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_3);+ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2, DATE_3);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_3);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2, DATE_3);++ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_3);+ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2, DATE_3);+ }++ @Test+ public void testAtomicUpdateAddDistinctOfDuplicateValueOnDateField() throws Exception {+ ensureFieldHasValues(COMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(COMMITTED_DOC_ID, "timestamps_mdt", DATE_2);+ ensureFieldHasValues(COMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);++ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(COMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_2);+ ensureFieldHasValues(COMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2);++ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_2);+ ensureFieldHasValues(UNCOMMITTED_DOC_ID, "timestamps_mdt", DATE_1, DATE_2);++ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2);+ atomicAddDistinctValueToField(UNCOMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_2);+ ensureFieldHasValues(UNCOMMITTED_DOC_STR_VALUES_ID, "timestamps_mdt", DATE_1, DATE_2);+ }++ private void atomicRemoveValueFromField(String docId, String fieldName, Object value) throws Exception {+ final SolrInputDocument doc = new SolrInputDocument();+ doc.setField("id", docId);+ Map<String, Object> atomicUpdateRemoval = new HashMap<>(1);+ atomicUpdateRemoval.put("remove", value);+ doc.setField(fieldName, atomicUpdateRemoval);++ cluster.getSolrClient().add(COLLECTION, doc);+ }++ private void atomicAddDistinctValueToField(String docId, String fieldName, Object value) throws Exception {+ final SolrInputDocument doc = new SolrInputDocument();+ doc.setField("id", docId);+ Map<String, Object> atomicUpdateRemoval = new HashMap<>(1);+ atomicUpdateRemoval.put("add-distinct", value);+ doc.setField(fieldName, atomicUpdateRemoval);++ cluster.getSolrClient().add(COLLECTION, doc);+ }++ private void ensureFieldHasValues(String identifyingDocId, String fieldName, Object... expectedValues) throws Exception {+ final ModifiableSolrParams solrParams = new ModifiableSolrParams();+ solrParams.set("id", identifyingDocId);+ QueryRequest request = new QueryRequest(solrParams);+ request.setPath("/get");+ final QueryResponse response = request.process(cluster.getSolrClient(), COLLECTION);++ final NamedList<Object> rawResponse = response.getResponse();+ assertTrue(rawResponse.get("doc") != null);+ assertTrue(rawResponse.get("doc") instanceof SolrDocument);+ final SolrDocument doc = (SolrDocument) rawResponse.get("doc");+ final Collection<Object> valuesAfterUpdate = doc.getFieldValues(fieldName);+ assertEquals("Expected field to have " + expectedValues.length + " values, but found " + valuesAfterUpdate.size(),+ expectedValues.length, valuesAfterUpdate.size());+ for (Object expectedValue: expectedValues) {+ assertTrue("Expected value [" + expectedValue + "] was not found in field", valuesAfterUpdate.contains(expectedValue));+ }+ }+}diff --git a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdateRemovalJavabinTest.java b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdateRemovalJavabinTest.javadeleted file mode 100644index c028051f644..00000000000--- a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdateRemovalJavabinTest.java+++ /dev/null@@ -1,132 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.update.processor;--import java.time.Instant;-import java.util.Collection;-import java.util.Date;-import java.util.HashMap;-import java.util.Map;--import org.apache.solr.client.solrj.SolrClient;-import org.apache.solr.client.solrj.SolrQuery;-import org.apache.solr.client.solrj.request.CollectionAdminRequest;-import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.client.solrj.response.QueryResponse;-import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.SolrDocumentList;-import org.apache.solr.common.SolrInputDocument;-import org.junit.BeforeClass;-import org.junit.Test;--/**- * Tests Solr's atomic-update functionality using requests sent through SolrJ using wt=javabin- *- * {@link AtomicUpdatesTest} covers some of the same functionality, but does so by making xml-based requests. Recent- * changes to Solr have made it possible for the same data sent with different formats to result in different NamedLists- * after unmarshalling, so the test duplication is now necessary. See SOLR-13331 for an example.- */-public class AtomicUpdateRemovalJavabinTest extends SolrCloudTestCase {- private static final String COLLECTION = "collection1";- private static final int NUM_SHARDS = 1;- private static final int NUM_REPLICAS = 1;- private static final Date DATE_1 = new Date();- private static final Date DATE_2 = Date.from(Instant.ofEpochSecond(1554243909));-- @BeforeClass- public static void setupCluster() throws Exception {- configureCluster(1)- .addConfig("conf", configset("cloud-dynamic"))- .configure();-- CollectionAdminRequest.createCollection(COLLECTION, "conf", NUM_SHARDS, NUM_REPLICAS)- .process(cluster.getSolrClient());-- cluster.waitForActiveCollection(COLLECTION, 1, 1);-- final SolrInputDocument doc1 = sdoc(- "id", "1",- "title_s", "title_1", "title_s", "title_2",- "tv_mv_text", "text_1", "tv_mv_text", "text_2",- "count_is", 1, "count_is", 2,- "count_md", 1.0, "count_md", 2.0,- "timestamps_mdt", DATE_1, "timestamps_mdt", DATE_2);- final UpdateRequest req = new UpdateRequest()- .add(doc1);- req.commit(cluster.getSolrClient(), COLLECTION);- }-- @Test- public void testAtomicUpdateRemovalOfStrField() throws Exception {- ensureFieldHasValues("1", "title_s", "title_1", "title_2");- atomicRemoveValueFromField("1", "title_s", "title_1");- ensureFieldHasValues("1", "title_s", "title_2");- }-- @Test- public void testAtomicUpdateRemovalOfTextField() throws Exception {- ensureFieldHasValues("1", "tv_mv_text", "text_1", "text_2");- atomicRemoveValueFromField("1", "tv_mv_text", "text_1");- ensureFieldHasValues("1", "tv_mv_text", "text_2");- }-- @Test- public void testAtomicUpdateRemovalOfIntField() throws Exception {- ensureFieldHasValues("1", "count_is", 1, 2);- atomicRemoveValueFromField("1", "count_is", 1);- ensureFieldHasValues("1", "count_is", 2);- }-- @Test- public void testAtomicUpdateRemovalOfDoubleField() throws Exception {- ensureFieldHasValues("1", "count_md", 1.0, 2.0);- atomicRemoveValueFromField("1", "count_md", 1.0);- ensureFieldHasValues("1", "count_md", 2.0);- }-- @Test- public void testAtomicUpdateRemovalOfDateField() throws Exception {- ensureFieldHasValues("1", "timestamps_mdt", DATE_1, DATE_2);- atomicRemoveValueFromField("1", "timestamps_mdt", DATE_1);- ensureFieldHasValues("1", "timestamps_mdt", DATE_2);- }-- private void atomicRemoveValueFromField(String docId, String fieldName, Object value) throws Exception {- final SolrInputDocument doc = new SolrInputDocument();- doc.setField("id", docId);- Map<String, Object> atomicUpdateRemoval = new HashMap<>(1);- atomicUpdateRemoval.put("remove", value);- doc.setField(fieldName, atomicUpdateRemoval);-- cluster.getSolrClient().add(COLLECTION, doc);- cluster.getSolrClient().commit(COLLECTION);- }-- private void ensureFieldHasValues(String identifyingDocId, String fieldName, Object... expectedValues) throws Exception {- final SolrClient client = cluster.getSolrClient();-- final QueryResponse response = client.query(COLLECTION, new SolrQuery("id:" + identifyingDocId));- final SolrDocumentList docs = response.getResults();- assertEquals(1, docs.getNumFound());- final Collection<Object> valuesAfterUpdate = docs.get(0).getFieldValues(fieldName);- assertEquals(expectedValues.length, valuesAfterUpdate.size());- for (Object expectedValue: expectedValues) {- assertTrue("Expected value [" + expectedValue + "] was not found in field", valuesAfterUpdate.contains(expectedValue));- }- }-}diff --git a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.javaindex 5a15e10daec..54e9f9d0e53 100644--- a/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java+++ b/solr/core/src/test/org/apache/solr/update/processor/AtomicUpdatesTest.java@@ -181,6 +181,22 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");assertQ(req("q", "intRemove:111", "indent", "true"), "//result[@numFound = '3']");++ // Test that mv int fields can have values removed prior to being committed to index (see SOLR-14971)+ doc = new SolrInputDocument();+ doc.setField("id", "4242");+ doc.setField("values_is", new String[] {"111", "222", "333"});+ assertU(adoc(doc));++ doc = new SolrInputDocument();+ doc.setField("id", "4242");+ doc.setField("values_is", ImmutableMap.of("remove", 111));+ assertU(adoc(doc));+ assertU(commit());++ assertQ(req("q", "values_is:111", "indent", "true"), "//result[@numFound = '0']");+ assertQ(req("q", "values_is:222", "indent", "true"), "//result[@numFound = '1']");+ assertQ(req("q", "values_is:333", "indent", "true"), "//result[@numFound = '1']");}@@ -251,6 +267,22 @@ public class AtomicUpdatesTest extends SolrTestCaseJ4 {assertQ(req("q", "intRemove:[* TO *]", "indent", "true"), "//result[@numFound = '4']");assertQ(req("q", "intRemove:111", "indent", "true"), "//result[@numFound = '3']");++ // Test that mv int fields can have values removed prior to being committed to index (see SOLR-14971)+ doc = new SolrInputDocument();+ doc.setField("id", "4242");+ doc.setField("values_is", new Integer[] {111, 222, 333});+ assertU(adoc(doc));++ doc = new SolrInputDocument();+ doc.setField("id", "4242");+ doc.setField("values_is", ImmutableMap.of("remove", 111));+ assertU(adoc(doc));+ assertU(commit());++ assertQ(req("q", "values_is:111", "indent", "true"), "//result[@numFound = '0']");+ assertQ(req("q", "values_is:222", "indent", "true"), "//result[@numFound = '1']");+ assertQ(req("q", "values_is:333", "indent", "true"), "//result[@numFound = '1']");}@Test