Apache Solr open-source search software
SOLR-17397: Fix SkipExistingDocumentsProcessor to handle child docume… (#2631)
Tim Owen8ddc3be74389
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 0fe7e1b8d02..8328743d647 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -107,7 +107,7 @@ New FeaturesImprovements----------------------(No changes)+* SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with child documents. (Tim Owens via Eric Pugh)Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.javaindex 31894b9fb49..4e55fc3e01b 100644--- a/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java+++ b/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java@@ -191,7 +191,7 @@ public class SkipExistingDocumentsProcessorFactory extends UpdateRequestProcessoreturn this.skipUpdateIfMissing;}- boolean doesDocumentExist(BytesRef indexedDocId) throws IOException {+ boolean doesDocumentExist(BytesRef indexedDocId) {assert null != indexedDocId;// we don't need any fields populated, we just need to know if the doc is in the tlog...@@ -224,6 +224,17 @@ public class SkipExistingDocumentsProcessorFactory extends UpdateRequestProcesso}}+ boolean doesChildDocumentExist(AddUpdateCommand cmd) throws IOException {+ return RealTimeGetComponent.getInputDocument(+ core,+ core.getLatestSchema().indexableUniqueKey(cmd.getSelfOrNestedDocIdStr()),+ cmd.getIndexedId(),+ null,+ null,+ RealTimeGetComponent.Resolution.DOC)+ != null;+ }+boolean isLeader(UpdateCommand cmd) {if ((cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0) {return false;@@ -241,17 +252,10 @@ public class SkipExistingDocumentsProcessorFactory extends UpdateRequestProcessoboolean isUpdate = AtomicUpdateDocumentMerger.isAtomicUpdate(cmd);- // boolean existsByLookup = (RealTimeGetComponent.getInputDocument(core, indexedDocId) !=- // null);- // if (docExists != existsByLookup) {- // log.error("Found docExists {} but existsByLookup {} for doc {}", docExists,- // existsByLookup, indexedDocId.utf8ToString());- // }-if (log.isDebugEnabled()) {log.debug("Document ID {} ... exists already? {} ... isAtomicUpdate? {} ... isLeader? {}",- indexedDocId.utf8ToString(),+ cmd.getPrintableId(),doesDocumentExist(indexedDocId),isUpdate,isLeader(cmd));@@ -259,20 +263,35 @@ public class SkipExistingDocumentsProcessorFactory extends UpdateRequestProcessoif (skipInsertIfExists && !isUpdate && isLeader(cmd) && doesDocumentExist(indexedDocId)) {if (log.isDebugEnabled()) {- log.debug("Skipping insert for pre-existing document ID {}", indexedDocId.utf8ToString());+ log.debug("Skipping insert for pre-existing document ID {}", cmd.getPrintableId());}return;}- if (skipUpdateIfMissing && isUpdate && isLeader(cmd) && !doesDocumentExist(indexedDocId)) {- if (log.isDebugEnabled()) {- log.debug("Skipping update to non-existent document ID {}", indexedDocId.utf8ToString());+ if (skipUpdateIfMissing && isUpdate && isLeader(cmd)) {+ if (!cmd.getSelfOrNestedDocIdStr().equals(cmd.getIndexedIdStr())) {+ // must be a child document+ if (!doesChildDocumentExist(cmd)) {+ if (log.isDebugEnabled()) {+ log.debug(+ "Skipping update to non-existent child document ID {}",+ cmd.getSelfOrNestedDocIdStr());+ }+ return;+ }+ } else {+ // not a child document+ if (!doesDocumentExist(indexedDocId)) {+ if (log.isDebugEnabled()) {+ log.debug("Skipping update to non-existent document ID {}", cmd.getPrintableId());+ }+ return;+ }}- return;}if (log.isDebugEnabled()) {- log.debug("Passing on document ID {}", indexedDocId.utf8ToString());+ log.debug("Passing on document ID {}", cmd.getPrintableId());}super.processAdd(cmd);diff --git a/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.javaindex bea8bd507a4..19752e1ac7f 100644--- a/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java+++ b/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java@@ -231,7 +231,7 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {}@Test- public void testSkippableInsertIsNotSkippedIfSkipInsertsFalse() throws IOException {+ public void testSkippableInsertIsNotSkippedIfSkipInsertsIsFalse() throws IOException {UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);SkipExistingDocumentsUpdateProcessor processor =Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, false));@@ -245,7 +245,7 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {}@Test- public void testSkippableInsertIsSkippedIfSkipInsertsTrue() throws IOException {+ public void testSkippableInsertIsSkippedIfSkipInsertsIsTrue() throws IOException {UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);SkipExistingDocumentsUpdateProcessor processor =Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, true, false));@@ -259,7 +259,7 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {}@Test- public void testNonSkippableInsertIsNotSkippedIfSkipInsertsTrue() throws IOException {+ public void testNonSkippableInsertIsNotSkippedIfSkipInsertsIsTrue() throws IOException {UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);SkipExistingDocumentsUpdateProcessor processor =Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, true, false));@@ -287,7 +287,7 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {}@Test- public void testSkippableUpdateIsNotSkippedIfSkipUpdatesFalse() throws IOException {+ public void testSkippableUpdateIsNotSkippedIfSkipUpdatesIsFalse() throws IOException {UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);SkipExistingDocumentsUpdateProcessor processor =Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, false));@@ -301,7 +301,7 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {}@Test- public void testSkippableUpdateIsSkippedIfSkipUpdatesTrue() throws IOException {+ public void testSkippableUpdateIsSkippedIfSkipUpdatesIsTrue() throws IOException {UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);SkipExistingDocumentsUpdateProcessor processor =Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true));@@ -315,7 +315,23 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {}@Test- public void testNonSkippableUpdateIsNotSkippedIfSkipUpdatesTrue() throws IOException {+ public void testSkippableChildDocUpdateIsSkippedIfSkipUpdatesIsTrue() throws IOException {+ UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);+ SkipExistingDocumentsUpdateProcessor processor =+ Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true));++ AddUpdateCommand cmd = Mockito.spy(createAtomicUpdateCmd(defaultRequest));+ doReturn(true).when(processor).isLeader(cmd);+ doReturn(false).when(processor).doesChildDocumentExist(cmd);+ doReturn("123/child1").when(cmd).getSelfOrNestedDocIdStr();+ doReturn("123").when(cmd).getIndexedIdStr();++ processor.processAdd(cmd);+ verify(next, never()).processAdd(cmd);+ }++ @Test+ public void testNonSkippableUpdateIsNotSkippedIfSkipUpdatesIsTrue() throws IOException {UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);SkipExistingDocumentsUpdateProcessor processor =Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true));@@ -328,6 +344,22 @@ public class SkipExistingDocumentsProcessorFactoryTest extends SolrTestCaseJ4 {verify(next).processAdd(cmd);}+ @Test+ public void testNonSkippableChildDocUpdateIsNotSkippedIfSkipUpdatesIsTrue() throws IOException {+ UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class);+ SkipExistingDocumentsUpdateProcessor processor =+ Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true));++ AddUpdateCommand cmd = Mockito.spy(createAtomicUpdateCmd(defaultRequest));+ doReturn(true).when(processor).isLeader(cmd);+ doReturn(true).when(processor).doesChildDocumentExist(cmd);+ doReturn("123/child1").when(cmd).getSelfOrNestedDocIdStr();+ doReturn("123").when(cmd).getIndexedIdStr();++ processor.processAdd(cmd);+ verify(next).processAdd(cmd);+ }+private AddUpdateCommand createInsertUpdateCmd(SolrQueryRequest req) {AddUpdateCommand cmd = new AddUpdateCommand(req);cmd.setIndexedId(docId);