Apache Lucene open-source search software
Store DocValuesSkippers in a separate file (#15976)
Alan Woodward37b6d65a7d6d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 7c17ef0d8e9..278a4ae28b7 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -299,6 +299,8 @@ Improvementsbuffer was controlled by an explicit parameter (set as `resultSimilarity` - `traversalSimilarity`), which is nowdeprecated. (Kaival Parikh)+* GITHUB#15976: Store DocValuesSkipper data in their own file. (Alan Woodward)+Optimizations---------------------* GITHUB#15861: Optimise PhraseScorer by short circuiting non competitive documents in TOP_SCORES mode. (Prithvi S)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.javaindex 8223320dfcc..833acb26a49 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java@@ -63,7 +63,7 @@ import org.apache.lucene.util.packed.DirectWriter;/** writer for {@link Lucene90DocValuesFormat} */final class Lucene90DocValuesConsumer extends DocValuesConsumer {- IndexOutput data, meta;+ IndexOutput data, meta, skipIndex;final int maxDoc;private byte[] termsDictBuffer;private final int skipIndexIntervalSize;@@ -75,7 +75,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {String dataCodec,String dataExtension,String metaCodec,- String metaExtension)+ String metaExtension,+ String skipIndexCodec,+ String skipIndexExtension)throws IOException {this.termsDictBuffer = new byte[1 << 14];try {@@ -99,6 +101,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {Lucene90DocValuesFormat.VERSION_CURRENT,state.segmentInfo.getId(),state.segmentSuffix);+ String skipIndexName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, skipIndexExtension);+ skipIndex = state.directory.createOutput(skipIndexName, state.context);+ CodecUtil.writeIndexHeader(+ skipIndex,+ skipIndexCodec,+ Lucene90DocValuesFormat.VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);maxDoc = state.segmentInfo.maxDoc();this.skipIndexIntervalSize = skipIndexIntervalSize;} catch (Throwable t) {@@ -118,13 +130,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {if (data != null) {CodecUtil.writeFooter(data); // write checksum}+ if (skipIndex != null) {+ CodecUtil.writeFooter(skipIndex); // write checksum+ }} catch (Throwable t) {- IOUtils.closeWhileSuppressingExceptions(t, data, meta);+ IOUtils.closeWhileSuppressingExceptions(t, data, meta, skipIndex);throw t;}- IOUtils.close(data, meta);+ IOUtils.close(data, meta, skipIndex);} finally {- meta = data = null;+ meta = data = skipIndex = null;}}@@ -246,7 +261,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer)throws IOException {assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;- final long start = data.getFilePointer();+ final long start = skipIndex.getFilePointer();final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);long globalMaxValue = Long.MIN_VALUE;long globalMinValue = Long.MAX_VALUE;@@ -288,7 +303,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {writeLevels(accumulators);}meta.writeLong(start); // record the start in meta- meta.writeLong(data.getFilePointer() - start); // record the length+ meta.writeLong(skipIndex.getFilePointer() - start); // record the lengthassert globalDocCount == 0 || globalMaxValue >= globalMinValue;meta.writeLong(globalMaxValue);meta.writeLong(globalMinValue);@@ -308,17 +323,17 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {// compute how many levels we need to write for the current accumulatorfinal int levels = getLevels(index, totalAccumulators);// write the number of levels- data.writeByte((byte) levels);+ skipIndex.writeByte((byte) levels);// write intervals in reverse order. This is done so we don't// need to read all of them in case of slippingfor (int level = levels - 1; level >= 0; level--) {final SkipAccumulator accumulator =accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level));- data.writeInt(accumulator.maxDocID);- data.writeInt(accumulator.minDocID);- data.writeLong(accumulator.maxValue);- data.writeLong(accumulator.minValue);- data.writeInt(accumulator.docCount);+ skipIndex.writeInt(accumulator.maxDocID);+ skipIndex.writeInt(accumulator.minDocID);+ skipIndex.writeLong(accumulator.maxValue);+ skipIndex.writeLong(accumulator.minValue);+ skipIndex.writeInt(accumulator.docCount);}}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.javaindex 922bc2481c9..cd4bb7ef076 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java@@ -132,6 +132,7 @@ import org.apache.lucene.util.packed.DirectWriter;* <ol>* <li><code>.dvd</code>: DocValues data* <li><code>.dvm</code>: DocValues metadata+ * <li><code>.dvs</code>: DocValues skip index (since version 1)* </ol>** @lucene.experimental@@ -158,21 +159,37 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {@Overridepublic DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {return new Lucene90DocValuesConsumer(- state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);+ state,+ skipIndexIntervalSize,+ DATA_CODEC,+ DATA_EXTENSION,+ META_CODEC,+ META_EXTENSION,+ SKIP_INDEX_CODEC,+ SKIP_INDEX_EXTENSION);}@Overridepublic DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {return new Lucene90DocValuesProducer(- state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);+ state,+ DATA_CODEC,+ DATA_EXTENSION,+ META_CODEC,+ META_EXTENSION,+ SKIP_INDEX_CODEC,+ SKIP_INDEX_EXTENSION);}static final String DATA_CODEC = "Lucene90DocValuesData";static final String DATA_EXTENSION = "dvd";static final String META_CODEC = "Lucene90DocValuesMetadata";static final String META_EXTENSION = "dvm";+ static final String SKIP_INDEX_CODEC = "Lucene90DocValuesSkipIndex";+ static final String SKIP_INDEX_EXTENSION = "dvs";static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;+ static final int VERSION_SKIPPER_SEPARATE_FILE = 1;+ static final int VERSION_CURRENT = VERSION_SKIPPER_SEPARATE_FILE;// indicates docvalues typestatic final byte NUMERIC = 0;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.javaindex 71fccdbf825..c20baa4b35e 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java@@ -66,6 +66,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {private final IntObjectHashMap<SortedNumericEntry> sortedNumerics;private final IntObjectHashMap<DocValuesSkipperEntry> skippers;private final IndexInput data;+ private final IndexInput skipIndexData;private final int maxDoc;private int version = -1;private final boolean merging;@@ -76,7 +77,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {String dataCodec,String dataExtension,String metaCodec,- String metaExtension)+ String metaExtension,+ String skipIndexCodec,+ String skipIndexExtension)throws IOException {String metaName =IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);@@ -139,6 +142,35 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {IOUtils.closeWhileSuppressingExceptions(t, data);throw t;}++ if (version >= Lucene90DocValuesFormat.VERSION_SKIPPER_SEPARATE_FILE) {+ String skipIndexName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, skipIndexExtension);+ this.skipIndexData =+ state.directory.openInput(skipIndexName, state.context.withHints(FileTypeHint.INDEX));+ try {+ final int skipVersion =+ CodecUtil.checkIndexHeader(+ skipIndexData,+ skipIndexCodec,+ Lucene90DocValuesFormat.VERSION_SKIPPER_SEPARATE_FILE,+ Lucene90DocValuesFormat.VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ if (version != skipVersion) {+ throw new CorruptIndexException(+ "Format versions mismatch: meta=" + version + ", skipIndex=" + skipVersion,+ skipIndexData);+ }+ CodecUtil.retrieveChecksum(skipIndexData);+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, data, skipIndexData);+ throw t;+ }+ } else {+ this.skipIndexData = null;+ }}// Used for cloning@@ -150,6 +182,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {IntObjectHashMap<SortedNumericEntry> sortedNumerics,IntObjectHashMap<DocValuesSkipperEntry> skippers,IndexInput data,+ IndexInput skipIndexData,int maxDoc,int version,boolean merging) {@@ -160,6 +193,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {this.sortedNumerics = sortedNumerics;this.skippers = skippers;this.data = data.clone();+ this.skipIndexData = skipIndexData != null ? skipIndexData.clone() : null;this.maxDoc = maxDoc;this.version = version;this.merging = merging;@@ -175,6 +209,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {sortedNumerics,skippers,data,+ skipIndexData,maxDoc,version,true);@@ -349,7 +384,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {@Overridepublic void close() throws IOException {- data.close();+ IOUtils.close(data, skipIndexData);}private record DocValuesSkipperEntry(@@ -1784,6 +1819,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {@Overridepublic void checkIntegrity() throws IOException {CodecUtil.checksumEntireFile(data);+ if (skipIndexData != null) {+ CodecUtil.checksumEntireFile(skipIndexData);+ }}/**@@ -1864,7 +1902,8 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {final DocValuesSkipperEntry entry = skippers.get(field.number);- final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);+ final IndexInput skipperSource = skipIndexData != null ? skipIndexData : data;+ final IndexInput input = skipperSource.slice("doc value skipper", entry.offset, entry.length);// TODO: should we write to disk the actual max level for this segment?return new DocValuesSkipper() {final int[] minDocID = new int[SKIP_INDEX_MAX_LEVEL];diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.javaindex 14f2a541703..50cb8b7aab3 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java@@ -45,6 +45,7 @@ import org.apache.lucene.document.StringField;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;@@ -64,6 +65,10 @@ import org.apache.lucene.index.TermsEnum.SeekStatus;import org.apache.lucene.store.ByteBuffersDataInput;import org.apache.lucene.store.ByteBuffersDataOutput;import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.codecs.asserting.AssertingCodec;import org.apache.lucene.tests.index.BaseCompressingDocValuesFormatTestCase;@@ -1020,4 +1025,52 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatTreader.close();directory.close();}++ public void testSkipIndexStoredSeparately() throws IOException {+ try (Directory dir = newDirectory()) {+ IndexWriterConfig conf = newIndexWriterConfig();+ conf.setCodec(getCodec());+ conf.setUseCompoundFile(false);+ try (IndexWriter writer = new IndexWriter(dir, conf)) {+ for (int i = 0; i < 100; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("dv", i));+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ }++ // Wrap the directory so that any attempt to slice() the .dvd file throws.+ // The producer constructor opens .dvd for header/checksum validation (sequential reads),+ // but getSkipper() should only slice the .dvs file, never the .dvd file.+ Directory noDataSliceDir =+ new FilterDirectory(dir) {+ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ IndexInput in = super.openInput(name, context);+ if (name.endsWith("." + Lucene90DocValuesFormat.DATA_EXTENSION)) {+ return new FilterIndexInput("no-slice-" + name, in) {+ @Override+ public IndexInput slice(String sliceDescription, long offset, long length) {+ throw new AssertionError(+ "should not slice .dvd file for skip index access: " + sliceDescription);+ }+ };+ }+ return in;+ }+ };++ try (DirectoryReader reader = DirectoryReader.open(noDataSliceDir)) {+ LeafReader leaf = getOnlyLeafReader(reader);+ DocValuesSkipper skipper = leaf.getDocValuesSkipper("dv");+ assertNotNull(skipper);+ assertEquals(100, skipper.docCount());+ skipper.advance(0);+ assertEquals(0, skipper.minDocID(0));+ assertTrue(skipper.maxDocID(0) >= 0);+ assertTrue(skipper.minValue() <= skipper.maxValue());+ }+ }+ }}