Apache Lucene open-source search software
Support deletions in rearrange (#11815)
Stefan Vodita369a70f28948
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5257c959e32..45b1c4944fb 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -37,6 +37,8 @@ API Changes* GITHUB#11840: Query rewrite now takes an IndexSearcher instead of IndexReader to enable concurrentrewriting. (Patrick Zhai)+* GITHUB#11814: Support deletions in IndexRearranger. (Stefan Vodita)+New Features---------------------diff --git a/lucene/misc/src/java/org/apache/lucene/misc/index/BinaryDocValueSelector.java b/lucene/misc/src/java/org/apache/lucene/misc/index/BinaryDocValueSelector.javaindex 082675f4dcc..c1a54b72476 100644--- a/lucene/misc/src/java/org/apache/lucene/misc/index/BinaryDocValueSelector.java+++ b/lucene/misc/src/java/org/apache/lucene/misc/index/BinaryDocValueSelector.java@@ -22,6 +22,7 @@ import java.io.Serializable;import java.util.ArrayList;import java.util.HashSet;import java.util.List;+import java.util.Set;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;@@ -30,52 +31,94 @@ import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.store.Directory;import org.apache.lucene.util.BitSet;import org.apache.lucene.util.Bits;+import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.FixedBitSet;-/** Select documents using binary doc values */+/**+ * Use this selector to rearrange an index where documents can be uniquely identified based on+ * {@link BinaryDocValues}+ */public class BinaryDocValueSelector implements IndexRearranger.DocumentSelector, Serializable {private final String field;- private final HashSet<String> keySet;+ private final Set<BytesRef> keySet;- public BinaryDocValueSelector(String field, HashSet<String> keySet) {+ public BinaryDocValueSelector(String field, Set<BytesRef> keySet) {this.field = field;this.keySet = keySet;}@Override- public BitSet getFilteredLiveDocs(CodecReader reader) throws IOException {+ public BitSet getFilteredDocs(CodecReader reader) throws IOException {BinaryDocValues binaryDocValues = reader.getBinaryDocValues(field);- Bits oldLiveDocs = reader.getLiveDocs();FixedBitSet bits = new FixedBitSet(reader.maxDoc());- for (int i = 0; i < reader.maxDoc(); i++) {- if (oldLiveDocs != null && oldLiveDocs.get(i) == false) {- continue;- }- if (binaryDocValues.advanceExact(i)- && keySet.contains(binaryDocValues.binaryValue().utf8ToString())) {- bits.set(i);+ for (int docid = 0; docid < reader.maxDoc(); docid++) {+ if (binaryDocValues.advanceExact(docid) && keySet.contains(binaryDocValues.binaryValue())) {+ bits.set(docid);}}return bits;}- public static List<IndexRearranger.DocumentSelector> createFromExistingIndex(+ /**+ * Create a selector for the deletes in an index, which can then be applied to a rearranged index+ *+ * @param field tells which {@link BinaryDocValues} are the unique key+ * @param directory where the original index is present+ * @return a deletes selector to be passed to {@link IndexRearranger}+ */+ public static IndexRearranger.DocumentSelector createDeleteSelectorFromIndex(String field, Directory directory) throws IOException {- List<IndexRearranger.DocumentSelector> selectors = new ArrayList<>();++ Set<BytesRef> keySet = new HashSet<>();+try (IndexReader reader = DirectoryReader.open(directory)) {for (LeafReaderContext context : reader.leaves()) {- HashSet<String> keySet = new HashSet<>();Bits liveDocs = context.reader().getLiveDocs();+ if (liveDocs == null) {+ continue;+ }+BinaryDocValues binaryDocValues = context.reader().getBinaryDocValues(field);- for (int i = 0; i < context.reader().maxDoc(); i++) {- if (liveDocs != null && liveDocs.get(i) == false) {+ for (int docid = 0; docid < context.reader().maxDoc(); docid++) {+ if (liveDocs.get(docid) == true) {continue;}- if (binaryDocValues.advanceExact(i)) {- keySet.add(binaryDocValues.binaryValue().utf8ToString());++ if (binaryDocValues.advanceExact(docid)) {+ keySet.add(BytesRef.deepCopyOf(binaryDocValues.binaryValue()));+ } else {+ throw new AssertionError("Document " + docid + " doesn't have key " + field);+ }+ }+ }+ }+ return new BinaryDocValueSelector(field, keySet);+ }++ /**+ * Create a list of selectors that will reproduce the index geometry when used with {@link+ * IndexRearranger}+ *+ * @param field tells which {@link BinaryDocValues} are the unique key+ * @param directory where the original index is present+ * @return a list of selectors to be passed to {@link IndexRearranger}+ */+ public static List<IndexRearranger.DocumentSelector> createLiveSelectorsFromIndex(+ String field, Directory directory) throws IOException {++ List<IndexRearranger.DocumentSelector> selectors = new ArrayList<>();++ try (IndexReader reader = DirectoryReader.open(directory)) {+ for (LeafReaderContext context : reader.leaves()) {+ Set<BytesRef> keySet = new HashSet<>();+ BinaryDocValues binaryDocValues = context.reader().getBinaryDocValues(field);++ for (int docid = 0; docid < context.reader().maxDoc(); docid++) {+ if (binaryDocValues.advanceExact(docid)) {+ keySet.add(BytesRef.deepCopyOf(binaryDocValues.binaryValue()));} else {- throw new AssertionError("Document don't have selected key");+ throw new AssertionError("Document " + docid + " doesn't have key " + field);}}selectors.add(new BinaryDocValueSelector(field, keySet));diff --git a/lucene/misc/src/java/org/apache/lucene/misc/index/IndexRearranger.java b/lucene/misc/src/java/org/apache/lucene/misc/index/IndexRearranger.javaindex c05a1a7b959..7c3bd53cece 100644--- a/lucene/misc/src/java/org/apache/lucene/misc/index/IndexRearranger.java+++ b/lucene/misc/src/java/org/apache/lucene/misc/index/IndexRearranger.java@@ -21,6 +21,7 @@ import java.io.IOException;import java.util.ArrayList;import java.util.List;import java.util.concurrent.Callable;+import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;@@ -43,7 +44,20 @@ import org.apache.lucene.util.NamedThreadFactory;/*** Copy and rearrange index according to document selectors, from input dir to output dir. Length of- * documentSelectors determines how many segments there will be+ * documentSelectors determines how many segments there will be.+ *+ * <p>Rearranging works in 3 steps: 1. Assume all docs in the original index are live and create the+ * rearranged index using the segment selectors. 2. Go through the rearranged index and apply+ * deletes requested by the deletes selector. 3. Reorder the segments to match the order of the+ * selectors and check the validity of the rearranged index.+ *+ * <p>NB: You can't produce segments that only contain deletes. If you select all documents in a+ * segment for deletion, the entire segment will be discarded.+ *+ * <p>Example use case: You are testing search performance after a change to indexing. You can index+ * the same content using the old and new indexers and then rearrange one of them to the shape of+ * the other. Using rearrange will give more accurate measurements, since you will not be+ * introducing noise from index geometry.** <p>TODO: another possible (faster) approach to do this is to manipulate FlushPolicy and* MergePolicy at indexing time to create small desired segments first and merge them accordingly@@ -54,59 +68,89 @@ import org.apache.lucene.util.NamedThreadFactory;public class IndexRearranger {protected final Directory input, output;protected final IndexWriterConfig config;- protected final List<DocumentSelector> documentSelectors;++ // Each of these selectors will produce a segment in the rearranged index.+ // The segments will appear in the index in the order of the selectors that produced them.+ protected final List<DocumentSelector> segmentSelectors;++ // Documents selected here will be marked for deletion in the rearranged index, but not merged+ // away.+ protected final DocumentSelector deletedDocsSelector;/**- * Constructor+ * All args constructor** @param input input dir* @param output output dir* @param config index writer config- * @param documentSelectors specify what document is desired in the rearranged index segments,- * each selector correspond to one segment+ * @param segmentSelectors specify which documents are desired in the rearranged index segments;+ * each selector corresponds to one segment+ * @param deletedDocsSelector specify which documents are to be marked for deletion in the+ * rearranged index; this selector should be thread-safe*/public IndexRearranger(Directory input,Directory output,IndexWriterConfig config,- List<DocumentSelector> documentSelectors) {+ List<DocumentSelector> segmentSelectors,+ DocumentSelector deletedDocsSelector) {this.input = input;this.output = output;this.config = config;- this.documentSelectors = documentSelectors;+ this.segmentSelectors = segmentSelectors;+ this.deletedDocsSelector = deletedDocsSelector;+ }++ /** Constructor with no deletes to apply */+ public IndexRearranger(+ Directory input,+ Directory output,+ IndexWriterConfig config,+ List<DocumentSelector> segmentSelectors) {+ this(input, output, config, segmentSelectors, null);}public void execute() throws Exception {- config.setMergePolicy(- NoMergePolicy.INSTANCE); // do not merge since one addIndexes call create one segment- try (IndexWriter writer = new IndexWriter(output, config);+ ExecutorService executor =+ Executors.newFixedThreadPool(+ Math.min(Runtime.getRuntime().availableProcessors(), segmentSelectors.size()),+ new NamedThreadFactory("rearranger"));++ IndexWriterConfig createSegmentsConfig = new IndexWriterConfig(config.getAnalyzer());+ IndexWriterConfig applyDeletesConfig = new IndexWriterConfig(config.getAnalyzer());++ // Do not merge - each addIndexes call creates one segment+ createSegmentsConfig.setMergePolicy(NoMergePolicy.INSTANCE);+ applyDeletesConfig.setMergePolicy(NoMergePolicy.INSTANCE);++ try (IndexWriter writer = new IndexWriter(output, createSegmentsConfig);IndexReader reader = DirectoryReader.open(input)) {- ExecutorService executor =- Executors.newFixedThreadPool(- Math.min(Runtime.getRuntime().availableProcessors(), documentSelectors.size()),- new NamedThreadFactory("rearranger"));- ArrayList<Future<Void>> futures = new ArrayList<>();- for (DocumentSelector record : documentSelectors) {- Callable<Void> addSegment =- () -> {- addOneSegment(writer, reader, record);- return null;- };- futures.add(executor.submit(addSegment));- }- for (Future<Void> future : futures) {- future.get();- }- executor.shutdown();+ createRearrangedIndex(writer, reader, segmentSelectors, executor);}+ finalizeRearrange(output, segmentSelectors);++ try (IndexWriter writer = new IndexWriter(output, applyDeletesConfig);+ IndexReader reader = DirectoryReader.open(writer)) {+ applyDeletes(writer, reader, deletedDocsSelector, executor);+ }+ executor.shutdown();+ }++ /**+ * Place segments in the order of their respective selectors and ensure the rearrange was+ * performed correctly.+ */+ private static void finalizeRearrange(Directory output, List<DocumentSelector> segmentSelectors)+ throws IOException {List<SegmentCommitInfo> ordered = new ArrayList<>();try (IndexReader reader = DirectoryReader.open(output)) {- for (DocumentSelector ds : documentSelectors) {+ for (DocumentSelector ds : segmentSelectors) {int foundLeaf = -1;for (LeafReaderContext context : reader.leaves()) {SegmentReader sr = (SegmentReader) context.reader();- int docFound = ds.getFilteredLiveDocs(sr).nextSetBit(0);+ int docFound = ds.getFilteredDocs(sr).nextSetBit(0);if (docFound != DocIdSetIterator.NO_MORE_DOCS) {+ // Each document can be mapped to one segment at mostif (foundLeaf != -1) {throw new IllegalStateException("Document selector "@@ -129,6 +173,32 @@ public class IndexRearranger {sis.commit(output);}+ /**+ * Create the rearranged index as described by the segment selectors. Assume all documents in the+ * original index are live.+ */+ private static void createRearrangedIndex(+ IndexWriter writer,+ IndexReader reader,+ List<DocumentSelector> selectors,+ ExecutorService executor)+ throws ExecutionException, InterruptedException {++ ArrayList<Future<Void>> futures = new ArrayList<>();+ for (DocumentSelector selector : selectors) {+ Callable<Void> addSegment =+ () -> {+ addOneSegment(writer, reader, selector);+ return null;+ };+ futures.add(executor.submit(addSegment));+ }++ for (Future<Void> future : futures) {+ future.get();+ }+ }+private static void addOneSegment(IndexWriter writer, IndexReader reader, DocumentSelector selector) throws IOException {CodecReader[] readers = new CodecReader[reader.leaves().size()];@@ -139,6 +209,42 @@ public class IndexRearranger {writer.addIndexes(readers);}+ private static void applyDeletes(+ IndexWriter writer, IndexReader reader, DocumentSelector selector, ExecutorService executor)+ throws ExecutionException, InterruptedException {+ if (selector == null) {+ // There are no deletes to be applied+ return;+ }++ ArrayList<Future<Void>> futures = new ArrayList<>();+ for (LeafReaderContext context : reader.leaves()) {+ Callable<Void> applyDeletesToSegment =+ () -> {+ applyDeletesToOneSegment(writer, (CodecReader) context.reader(), selector);+ return null;+ };+ futures.add(executor.submit(applyDeletesToSegment));+ }++ for (Future<Void> future : futures) {+ future.get();+ }+ }++ private static void applyDeletesToOneSegment(+ IndexWriter writer, CodecReader segmentReader, DocumentSelector selector) throws IOException {+ Bits deletedDocs = selector.getFilteredDocs(segmentReader);+ for (int docid = 0; docid < segmentReader.maxDoc(); ++docid) {+ if (deletedDocs.get(docid)) {+ if (writer.tryDeleteDocument(segmentReader, docid) == -1) {+ throw new IllegalStateException(+ "tryDeleteDocument has failed. This should never happen, since merging is disabled.");+ }+ }+ }+ }+private static class DocSelectorFilteredCodecReader extends FilterCodecReader {BitSet filteredLiveDocs;@@ -147,7 +253,7 @@ public class IndexRearranger {public DocSelectorFilteredCodecReader(CodecReader in, DocumentSelector selector)throws IOException {super(in);- filteredLiveDocs = selector.getFilteredLiveDocs(in);+ filteredLiveDocs = selector.getFilteredDocs(in);numDocs = filteredLiveDocs.cardinality();}@@ -174,6 +280,6 @@ public class IndexRearranger {/** Select document within a CodecReader */public interface DocumentSelector {- BitSet getFilteredLiveDocs(CodecReader reader) throws IOException;+ BitSet getFilteredDocs(CodecReader reader) throws IOException;}}diff --git a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBinaryDocValueSelector.java b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBinaryDocValueSelector.javanew file mode 100644index 00000000000..4576c571ba3--- /dev/null+++ b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBinaryDocValueSelector.java@@ -0,0 +1,123 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.misc.index;++import java.io.IOException;+import org.apache.lucene.document.BinaryDocValuesField;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.BytesRef;++/**+ * This test creates an index with 100 documents and 10 segments where all the documents have a+ * binary doc value, "textOrd", which is filled with consecutive integers. The documents with even+ * "textOrd" are marked for deletion. The index is then rearranged into 4 segments using {@link+ * BinaryDocValueSelector}.+ */+public class TestBinaryDocValueSelector extends LuceneTestCase {+ public void testRearrangeUsingBinaryDocValueSelector() throws Exception {+ Directory srcDir = newDirectory();+ createIndex(100, 10, srcDir);+ assertSequentialIndex(srcDir, 100, 10);++ Directory inputDir = newDirectory();+ createIndex(100, 4, inputDir);+ assertSequentialIndex(inputDir, 100, 4);++ Directory outputDir = newDirectory();+ IndexRearranger rearranger =+ new IndexRearranger(+ inputDir,+ outputDir,+ getIndexWriterConfig(),+ BinaryDocValueSelector.createLiveSelectorsFromIndex("textOrd", srcDir),+ BinaryDocValueSelector.createDeleteSelectorFromIndex("textOrd", srcDir));+ rearranger.execute();+ assertSequentialIndex(outputDir, 100, 10);++ outputDir.close();+ inputDir.close();+ srcDir.close();+ }++ private static void assertSequentialIndex(Directory dir, int docNum, int segNum)+ throws IOException {+ IndexReader reader = DirectoryReader.open(dir);+ long lastOrd = -1;+ for (int i = 0; i < segNum; i++) {+ LeafReader leafReader = reader.leaves().get(i).reader();+ NumericDocValues numericDocValues = leafReader.getNumericDocValues("ord");+ Bits liveDocs = leafReader.getLiveDocs();+ assertNotNull(liveDocs);++ for (int doc = 0; doc < leafReader.maxDoc(); doc++) {+ assertTrue(numericDocValues.advanceExact(doc));+ assertEquals(numericDocValues.longValue(), lastOrd + 1);+ lastOrd = numericDocValues.longValue();++ assertEquals(liveDocs.get(doc), lastOrd % 2 != 0);+ }+ }+ assertEquals(docNum, lastOrd + 1);+ reader.close();+ }++ private static IndexWriterConfig getIndexWriterConfig() {+ return new IndexWriterConfig(null)+ .setOpenMode(IndexWriterConfig.OpenMode.CREATE)+ .setMergePolicy(NoMergePolicy.INSTANCE)+ .setIndexSort(new Sort(new SortField("ord", SortField.Type.INT)));+ }++ private static void createIndex(int docNum, int segNum, Directory dir) throws IOException {+ IndexWriter w = new IndexWriter(dir, getIndexWriterConfig());+ int docPerSeg = (int) Math.ceil((double) docNum / segNum);+ for (int i = 0; i < docNum; i++) {+ Document doc = new Document();+ doc.add(new NumericDocValuesField("ord", i));+ doc.add(new BinaryDocValuesField("textOrd", new BytesRef(Integer.toString(i))));+ if (i % 2 == 0) {+ doc.add(new StringField("delete", "yes", Field.Store.YES));+ }+ w.addDocument(doc);+ if (i % docPerSeg == docPerSeg - 1) {+ w.deleteDocuments(new Term("delete", "yes"));+ w.commit();+ }+ }+ IndexReader reader = DirectoryReader.open(w);+ assertEquals(segNum, reader.leaves().size());+ reader.close();+ w.close();+ }+}diff --git a/lucene/misc/src/test/org/apache/lucene/misc/index/TestIndexRearranger.java b/lucene/misc/src/test/org/apache/lucene/misc/index/TestIndexRearranger.javaindex afa75a3f330..3907b19f13c 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/index/TestIndexRearranger.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/index/TestIndexRearranger.java@@ -18,10 +18,12 @@package org.apache.lucene.misc.index;import java.io.IOException;+import java.util.Arrays;import java.util.List;-import org.apache.lucene.document.BinaryDocValuesField;import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.StringField;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;@@ -30,6 +32,7 @@ import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.NoMergePolicy;import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.Term;import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.lucene.store.Directory;@@ -39,10 +42,47 @@ import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.FixedBitSet;+/**+ * This test creates the following index:+ *+ * <p>segment 0: (ord = 0, live = true), (ord = 1, live = false)+ *+ * <p>segment 1: (ord = 2, live = true), (ord = 3, live = false)+ *+ * <p>segment 4: (ord = 4, live = true), (ord = 5, live = false)+ *+ * <p>It also creates 3 selectors:+ *+ * <p>{@link LiveToLiveSelector} selects for ord in {0, 2}+ *+ * <p>{@link DeleteToLiveSelector} selects for ord in {3, 5}+ *+ * <p>{@link DeleteSelector} selects for ord in {1, 2, 4, 5}+ */public class TestIndexRearranger extends LuceneTestCase {- public void testRearrange() throws Exception {+ /**+ * Use {@link LiveToLiveSelector} and {@link DeleteToLiveSelector} to rearrange the index into 2+ * segments:+ *+ * <p>segment 0: (ord = 0, live = true), (ord = 2, live = true)+ *+ * <p>segment 1: (ord = 3, live = true), (ord = 5, live = true)+ *+ * <p>The documents with ord 1 and 4 have now been lost, since they were not selected to be in the+ * rearranged index. All documents that were selected are now considered live.+ *+ * <p>Next, we apply the deletions specified by {@link DeleteSelector}:+ *+ * <p>segment 0: (ord = 0, live = true), (ord = 2, live = false)+ *+ * <p>segment 1: (ord = 3, live = true), (ord = 5, live = false)+ *+ * <p>The documents with ord 2 and 5 have been marked for deletion. Documents 1 and 4 would have+ * also been marked if they were present.+ */+ public void testLiveDeleteCombinations() throws Exception {Directory inputDir = newDirectory();- createIndex(100, 10, inputDir);+ createIndex(inputDir);Directory outputDir = newDirectory();IndexRearranger rearranger =@@ -50,42 +90,55 @@ public class TestIndexRearranger extends LuceneTestCase {inputDir,outputDir,getIndexWriterConfig(),- List.of(new OddDocSelector(), new EvenDocSelector()));+ List.of(+ new TestIndexRearranger.LiveToLiveSelector(),+ new TestIndexRearranger.DeleteToLiveSelector()),+ new TestIndexRearranger.DeleteSelector());rearranger.execute();+IndexReader reader = DirectoryReader.open(outputDir);assertEquals(2, reader.leaves().size());+LeafReader segment1 = reader.leaves().get(0).reader();- assertEquals(50, segment1.numDocs());+ assertEquals(1, segment1.numDocs());+ assertEquals(2, segment1.maxDoc());+ Bits liveDocs = segment1.getLiveDocs();+ assertNotNull(liveDocs);NumericDocValues numericDocValues = segment1.getNumericDocValues("ord");+ assertNotNull(numericDocValues);assertTrue(numericDocValues.advanceExact(0));- boolean odd = numericDocValues.longValue() % 2 == 1;- for (int i = 1; i < 50; i++) {- assertTrue(numericDocValues.advanceExact(i));- assertEquals(odd, numericDocValues.longValue() % 2 == 1);- }+ assertTrue(liveDocs.get(0));+ assertEquals(0, numericDocValues.longValue());+ assertTrue(numericDocValues.advanceExact(1));+ assertFalse(liveDocs.get(1));+ assertEquals(2, numericDocValues.longValue());+LeafReader segment2 = reader.leaves().get(1).reader();- assertEquals(50, segment2.numDocs());+ assertEquals(1, segment2.numDocs());+ assertEquals(2, segment2.maxDoc());+ liveDocs = segment1.getLiveDocs();+ assertNotNull(liveDocs);numericDocValues = segment2.getNumericDocValues("ord");+ assertNotNull(numericDocValues);+ assertTrue(liveDocs.get(0));assertTrue(numericDocValues.advanceExact(0));- boolean odd2 = numericDocValues.longValue() % 2 == 1;- assertTrue(odd != odd2);- for (int i = 1; i < 50; i++) {- assertTrue(numericDocValues.advanceExact(i));- assertEquals(odd2, numericDocValues.longValue() % 2 == 1);- }+ assertEquals(3, numericDocValues.longValue());+ assertFalse(liveDocs.get(1));+ assertTrue(numericDocValues.advanceExact(1));+ assertEquals(5, numericDocValues.longValue());+reader.close();inputDir.close();outputDir.close();}- public void testRearrangeUsingBinaryDocValueSelector() throws Exception {- Directory srcDir = newDirectory();- createIndex(100, 10, srcDir);- assertSequentialIndex(srcDir, 100, 10);-+ /**+ * This test arrives at an empty rearranged index by using the same selector for creating segments+ * and applying deletes.+ */+ public void testDeleteEverything() throws Exception {Directory inputDir = newDirectory();- createIndex(100, 4, inputDir);- assertSequentialIndex(inputDir, 100, 4);+ createIndex(inputDir);Directory outputDir = newDirectory();IndexRearranger rearranger =@@ -93,31 +146,64 @@ public class TestIndexRearranger extends LuceneTestCase {inputDir,outputDir,getIndexWriterConfig(),- BinaryDocValueSelector.createFromExistingIndex("textOrd", srcDir));+ List.of(new TestIndexRearranger.LiveToLiveSelector()),+ new TestIndexRearranger.LiveToLiveSelector());rearranger.execute();- assertSequentialIndex(outputDir, 100, 10);- outputDir.close();+ IndexReader reader = DirectoryReader.open(outputDir);+ assertEquals(0, reader.leaves().size());++ reader.close();inputDir.close();- srcDir.close();+ outputDir.close();}- private static void assertSequentialIndex(Directory dir, int docNum, int segNum)- throws IOException {- IndexReader reader = DirectoryReader.open(dir);- long lastOrd = -1;- for (int i = 0; i < segNum; i++) {- LeafReader leafReader = reader.leaves().get(i).reader();- NumericDocValues numericDocValues = leafReader.getNumericDocValues("ord");-- for (int doc = 0; doc < leafReader.numDocs(); doc++) {- assertTrue(numericDocValues.advanceExact(doc));- assertEquals(numericDocValues.longValue(), lastOrd + 1);- lastOrd = numericDocValues.longValue();- }- }- assertEquals(docNum, lastOrd + 1);+ /** Don't pass a deletes selector, all selected docs will be live. */+ public void testDeleteNothing() throws Exception {+ Directory inputDir = newDirectory();+ createIndex(inputDir);++ Directory outputDir = newDirectory();+ IndexRearranger rearranger =+ new IndexRearranger(+ inputDir,+ outputDir,+ getIndexWriterConfig(),+ List.of(+ new TestIndexRearranger.LiveToLiveSelector(),+ new TestIndexRearranger.DeleteToLiveSelector()));+ rearranger.execute();++ IndexReader reader = DirectoryReader.open(outputDir);+ assertEquals(2, reader.leaves().size());++ LeafReader segment1 = reader.leaves().get(0).reader();+ assertEquals(2, segment1.numDocs());+ assertEquals(2, segment1.maxDoc());+ Bits liveDocs = segment1.getLiveDocs();+ assertNull(liveDocs);+ NumericDocValues numericDocValues = segment1.getNumericDocValues("ord");+ assertNotNull(numericDocValues);+ assertTrue(numericDocValues.advanceExact(0));+ assertEquals(0, numericDocValues.longValue());+ assertTrue(numericDocValues.advanceExact(1));+ assertEquals(2, numericDocValues.longValue());++ LeafReader segment2 = reader.leaves().get(1).reader();+ assertEquals(2, segment2.numDocs());+ assertEquals(2, segment2.maxDoc());+ liveDocs = segment1.getLiveDocs();+ assertNull(liveDocs);+ numericDocValues = segment2.getNumericDocValues("ord");+ assertNotNull(numericDocValues);+ assertTrue(numericDocValues.advanceExact(0));+ assertEquals(3, numericDocValues.longValue());+ assertTrue(numericDocValues.advanceExact(1));+ assertEquals(5, numericDocValues.longValue());+reader.close();+ inputDir.close();+ outputDir.close();}private static IndexWriterConfig getIndexWriterConfig() {@@ -127,56 +213,92 @@ public class TestIndexRearranger extends LuceneTestCase {.setIndexSort(new Sort(new SortField("ord", SortField.Type.INT)));}- private static void createIndex(int docNum, int segNum, Directory dir) throws IOException {+ private static void createIndex(Directory dir) throws IOException {IndexWriter w = new IndexWriter(dir, getIndexWriterConfig());- int docPerSeg = (int) Math.ceil((double) docNum / segNum);- for (int i = 0; i < docNum; i++) {+ for (int i = 0; i < 6; i++) {Document doc = new Document();doc.add(new NumericDocValuesField("ord", i));- doc.add(new BinaryDocValuesField("textOrd", new BytesRef(Integer.toString(i))));+ if (i % 2 == 1) {+ doc.add(new StringField("delete", new BytesRef("yes"), Field.Store.YES));+ }w.addDocument(doc);- if (i % docPerSeg == docPerSeg - 1) {+ if (i % 2 == 1) {+ w.deleteDocuments(new Term("delete", "yes"));w.commit();}}- IndexReader reader = DirectoryReader.open(w);- assertEquals(segNum, reader.leaves().size());- reader.close();++ assertCreatedIndex(dir);w.close();}- private static class OddDocSelector implements IndexRearranger.DocumentSelector {+ private static void assertCreatedIndex(Directory dir) throws IOException {+ IndexReader reader = DirectoryReader.open(dir);+ assertEquals(3, reader.leaves().size());++ for (int i = 0; i < 3; i++) {+ LeafReader segmentReader = reader.leaves().get(i).reader();+ assertEquals(1, segmentReader.numDocs());+ assertEquals(2, segmentReader.maxDoc());+ Bits liveDocs = segmentReader.getLiveDocs();+ assertNotNull(liveDocs);+ assertTrue(liveDocs.get(0));+ assertFalse(liveDocs.get(1));++ NumericDocValues ord = segmentReader.getNumericDocValues("ord");+ assertNotNull(ord);+ assertTrue(ord.advanceExact(0));+ assertEquals(2 * i, ord.longValue());+ assertTrue(ord.advanceExact(1));+ assertEquals(2 * i + 1, ord.longValue());+ }++ reader.close();+ }++ private static class LiveToLiveSelector implements IndexRearranger.DocumentSelector {@Override- public BitSet getFilteredLiveDocs(CodecReader reader) throws IOException {+ public BitSet getFilteredDocs(CodecReader reader) throws IOException {FixedBitSet filteredSet = new FixedBitSet(reader.maxDoc());- Bits liveDocs = reader.getLiveDocs();NumericDocValues numericDocValues = reader.getNumericDocValues("ord");- for (int i = 0; i < reader.maxDoc(); i++) {- if (liveDocs != null && liveDocs.get(i) == false) {- continue;- }- if (numericDocValues.advanceExact(i) && numericDocValues.longValue() % 2 == 1) {- filteredSet.set(i);+ assert numericDocValues != null;+ for (int docid = 0; docid < reader.maxDoc(); docid++) {+ if (numericDocValues.advanceExact(docid)+ && Arrays.asList(0, 2).contains((int) numericDocValues.longValue())) {+ filteredSet.set(docid);}}return filteredSet;}}- private static class EvenDocSelector implements IndexRearranger.DocumentSelector {-+ private static class DeleteToLiveSelector implements IndexRearranger.DocumentSelector {@Override- public BitSet getFilteredLiveDocs(CodecReader reader) throws IOException {+ public BitSet getFilteredDocs(CodecReader reader) throws IOException {FixedBitSet filteredSet = new FixedBitSet(reader.maxDoc());- Bits liveDocs = reader.getLiveDocs();NumericDocValues numericDocValues = reader.getNumericDocValues("ord");- for (int i = 0; i < reader.maxDoc(); i++) {- if (liveDocs != null && liveDocs.get(i) == false) {- continue;+ assert numericDocValues != null;+ for (int docid = 0; docid < reader.maxDoc(); docid++) {+ if (numericDocValues.advanceExact(docid)+ && Arrays.asList(3, 5).contains((int) numericDocValues.longValue())) {+ filteredSet.set(docid);}- if (numericDocValues.advanceExact(i) && numericDocValues.longValue() % 2 == 0) {- filteredSet.set(i);+ }+ return filteredSet;+ }+ }++ private static class DeleteSelector implements IndexRearranger.DocumentSelector {+ @Override+ public BitSet getFilteredDocs(CodecReader reader) throws IOException {+ FixedBitSet filteredSet = new FixedBitSet(reader.maxDoc());+ NumericDocValues numericDocValues = reader.getNumericDocValues("ord");+ assert numericDocValues != null;+ for (int docid = 0; docid < reader.maxDoc(); docid++) {+ if (numericDocValues.advanceExact(docid)+ && Arrays.asList(1, 2, 4, 5).contains((int) numericDocValues.longValue())) {+ filteredSet.set(docid);}}return filteredSet;