Apache Lucene open-source search software
Store doc values disi data into a separate file (#16540)
Sagar Upadhyaya1160d3c8a256
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5944d5453a6..2cdc8ebdf2e 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -404,6 +404,8 @@ New FeaturesImprovements---------------------+* GITHUB#16298: Store doc values DISI data into a separate file. (Sagar Upadhyaya)+* GITHUB#16247: Introduce CachingCollectorManager to parallelize search when using CachingCollector.Remove experimental GroupingSearch constructor that takes a GroupSelector as argumentin favour of providing a GroupSelector supplier. (Binlong Gao)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.javaindex a7dff6b9e2c..5bdd5d9fba9 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java@@ -64,7 +64,7 @@ import org.apache.lucene.util.packed.DirectWriter;/** writer for {@link Lucene90DocValuesFormat} */final class Lucene90DocValuesConsumer extends DocValuesConsumer {- IndexOutput data, meta, skipIndex;+ IndexOutput data, meta, skipIndex, disiData;final int maxDoc;private byte[] termsDictBuffer;private final int skipIndexIntervalSize;@@ -78,7 +78,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {String metaCodec,String metaExtension,String skipIndexCodec,- String skipIndexExtension)+ String skipIndexExtension,+ String disiCodec,+ String disiExtension)throws IOException {this.termsDictBuffer = new byte[1 << 14];try {@@ -112,6 +114,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {Lucene90DocValuesFormat.VERSION_CURRENT,state.segmentInfo.getId(),state.segmentSuffix);+ String disiName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, disiExtension);+ disiData = state.directory.createOutput(disiName, state.context);+ CodecUtil.writeIndexHeader(+ disiData,+ disiCodec,+ Lucene90DocValuesFormat.VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);maxDoc = state.segmentInfo.maxDoc();this.skipIndexIntervalSize = skipIndexIntervalSize;} catch (Throwable t) {@@ -134,16 +146,33 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {if (skipIndex != null) {CodecUtil.writeFooter(skipIndex); // write checksum}+ if (disiData != null) {+ CodecUtil.writeFooter(disiData); // write checksum+ }} catch (Throwable t) {- IOUtils.closeWhileSuppressingExceptions(t, data, meta, skipIndex);+ IOUtils.closeWhileSuppressingExceptions(t, data, meta, skipIndex, disiData);throw t;}- IOUtils.close(data, meta, skipIndex);+ IOUtils.close(data, meta, skipIndex, disiData);} finally {- meta = data = skipIndex = null;+ meta = data = skipIndex = disiData = null;}}+ /**+ * Writes a sparse field's {@link IndexedDISI} to the {@code .dvp} file and records its offset,+ * length and shape in the metadata.+ */+ private void writeDISI(DocIdSetIterator values) throws IOException {+ final long offset = disiData.getFilePointer();+ meta.writeLong(offset); // docsWithFieldOffset (into .dvp)+ final short jumpTableEntryCount =+ IndexedDISI.writeBitSet(values, disiData, IndexedDISI.DEFAULT_DENSE_RANK_POWER);+ meta.writeLong(disiData.getFilePointer() - offset); // docsWithFieldLength (region in .dvp)+ meta.writeShort(jumpTableEntryCount);+ meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);+ }+@Overridepublic void addNumericField(FieldInfo field, DocValuesProducer valuesProducer)throws IOException {@@ -440,15 +469,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {meta.writeLong(0L); // docsWithFieldLengthmeta.writeShort((short) -1); // jumpTableEntryCountmeta.writeByte((byte) -1); // denseRankPower- } else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values- long offset = data.getFilePointer();- meta.writeLong(offset); // docsWithFieldOffset+ } else { // meta[disi.offset, disi.length]: IndexedDISI region in .dvp for documents with valuesvalues = valuesProducer.getSortedNumeric(field);- final short jumpTableEntryCount =- IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);- meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength- meta.writeShort(jumpTableEntryCount);- meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);+ writeDISI(values);}meta.writeLong(numValues);@@ -633,14 +656,8 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {meta.writeShort((short) -1); // jumpTableEntryCountmeta.writeByte((byte) -1); // denseRankPower} else {- long offset = data.getFilePointer();- meta.writeLong(offset); // docsWithFieldOffsetvalues = valuesProducer.getBinary(field);- final short jumpTableEntryCount =- IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);- meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength- meta.writeShort(jumpTableEntryCount);- meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);+ writeDISI(values);}meta.writeInt(numDocsWithField);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.javaindex 22cd7581f1d..777e76d9792 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java@@ -133,6 +133,8 @@ import org.apache.lucene.util.packed.DirectWriter;* <li><code>.dvd</code>: DocValues data* <li><code>.dvm</code>: DocValues metadata* <li><code>.dvs</code>: DocValues skip index (since version 1)+ * <li><code>.dvp</code>: DocValues presence, i.e. the {@link IndexedDISI} for sparse fields+ * (since version 3)* </ol>** @lucene.experimental@@ -166,7 +168,9 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {META_CODEC,META_EXTENSION,SKIP_INDEX_CODEC,- SKIP_INDEX_EXTENSION);+ SKIP_INDEX_EXTENSION,+ DISI_CODEC,+ DISI_EXTENSION);}@Override@@ -178,7 +182,9 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {META_CODEC,META_EXTENSION,SKIP_INDEX_CODEC,- SKIP_INDEX_EXTENSION);+ SKIP_INDEX_EXTENSION,+ DISI_CODEC,+ DISI_EXTENSION);}static final String DATA_CODEC = "Lucene90DocValuesData";@@ -187,10 +193,14 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {static final String META_EXTENSION = "dvm";static final String SKIP_INDEX_CODEC = "Lucene90DocValuesSkipIndex";static final String SKIP_INDEX_EXTENSION = "dvs";+ static final String DISI_CODEC = "Lucene90DocValuesDISI";+ static final String DISI_EXTENSION = "dvp";static final int VERSION_START = 0;static final int VERSION_SKIPPER_SEPARATE_FILE = 1;static final int VERSION_SKIPPER_MAX_VALUE_COUNT = 2;- static final int VERSION_CURRENT = VERSION_SKIPPER_MAX_VALUE_COUNT;+ // Moves the sparse-field IndexedDISI from .dvd into a dedicated .dvp file.+ static final int VERSION_DISI_SEPARATE_FILE = 3;+ static final int VERSION_CURRENT = VERSION_DISI_SEPARATE_FILE;// indicates docvalues typestatic final byte NUMERIC = 0;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.javaindex cdefbca58ff..dd67e555e5e 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java@@ -69,6 +69,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {private final IntObjectHashMap<DocValuesSkipperEntry> skippers;private final IndexInput data;private final IndexInput skipIndexData;+ // Sparse-field presence (IndexedDISI) file. Null for pre-VERSION_DISI_SEPARATE_FILE segments,+ // which still keep the IndexedDISI in .dvd.+ private final IndexInput disiData;private final int maxDoc;private int version = -1;private final boolean merging;@@ -81,7 +84,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {String metaCodec,String metaExtension,String skipIndexCodec,- String skipIndexExtension)+ String skipIndexExtension,+ String disiCodec,+ String disiExtension)throws IOException {String metaName =IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);@@ -178,6 +183,35 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {} else {this.skipIndexData = null;}++ if (version >= Lucene90DocValuesFormat.VERSION_DISI_SEPARATE_FILE) {+ IndexInput disiIn = null;+ try {+ String disiName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, disiExtension);+ disiIn = state.directory.openInput(disiName, state.context.withHints(FileTypeHint.INDEX));+ final int disiVersion =+ CodecUtil.checkIndexHeader(+ disiIn,+ disiCodec,+ Lucene90DocValuesFormat.VERSION_DISI_SEPARATE_FILE,+ Lucene90DocValuesFormat.VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ if (version != disiVersion) {+ throw new CorruptIndexException(+ "Format versions mismatch: meta=" + version + ", disi=" + disiVersion, disiIn);+ }+ CodecUtil.retrieveChecksum(disiIn);+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, data, skipIndexData, disiIn);+ throw t;+ }+ this.disiData = disiIn;+ } else {+ this.disiData = null;+ }}// Used for cloning@@ -190,6 +224,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {IntObjectHashMap<DocValuesSkipperEntry> skippers,IndexInput data,IndexInput skipIndexData,+ IndexInput disiData,int maxDoc,int version,boolean merging) {@@ -201,6 +236,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {this.skippers = skippers;this.data = data.clone();this.skipIndexData = skipIndexData != null ? skipIndexData.clone() : null;+ this.disiData = disiData != null ? disiData.clone() : null;this.maxDoc = maxDoc;this.version = version;this.merging = merging;@@ -217,11 +253,36 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {skippers,data,skipIndexData,+ disiData,maxDoc,version,true);}+ /**+ * Builds a sparse field's {@link IndexedDISI}.+ *+ * <p>Since {@link Lucene90DocValuesFormat#VERSION_DISI_SEPARATE_FILE} the bytes live in the+ * {@code .dvp} file; for older segments they are inline in {@code .dvd}. The offset, length and+ * shape are read from the metadata in both cases.+ */+ private IndexedDISI newIndexedDISI(+ long docsWithFieldOffset,+ long docsWithFieldLength,+ short jumpTableEntryCount,+ byte denseRankPower,+ long numValues)+ throws IOException {+ final IndexInput in = disiData != null ? disiData : data;+ return new IndexedDISI(+ in,+ docsWithFieldOffset,+ docsWithFieldLength,+ jumpTableEntryCount,+ denseRankPower,+ numValues);+ }+private void inferMaxValueCounts(FieldInfos fieldInfos) {for (var cursor : skippers) {DocValuesSkipperEntry entry = cursor.value;@@ -447,7 +508,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {@Overridepublic void close() throws IOException {- IOUtils.close(data, skipIndexData);+ IOUtils.close(data, skipIndexData, disiData);}private record DocValuesSkipperEntry(@@ -1020,8 +1081,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {} else {// sparsefinal IndexedDISI disi =- new IndexedDISI(- data,+ newIndexedDISI(entry.docsWithFieldOffset,entry.docsWithFieldLength,entry.jumpTableEntryCount,@@ -1359,8 +1419,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {} else {// sparsefinal IndexedDISI disi =- new IndexedDISI(- data,+ newIndexedDISI(entry.docsWithFieldOffset,entry.docsWithFieldLength,entry.jumpTableEntryCount,@@ -1497,8 +1556,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {};} else if (ordsEntry.docsWithFieldOffset >= 0) { // sparse but non-emptyfinal IndexedDISI disi =- new IndexedDISI(- data,+ newIndexedDISI(ordsEntry.docsWithFieldOffset,ordsEntry.docsWithFieldLength,ordsEntry.jumpTableEntryCount,@@ -2087,8 +2145,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {} else {// sparsefinal IndexedDISI disi =- new IndexedDISI(- data,+ newIndexedDISI(entry.docsWithFieldOffset,entry.docsWithFieldLength,entry.jumpTableEntryCount,@@ -2324,8 +2381,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {};} else if (ordsEntry.docsWithFieldOffset >= 0) { // sparse but non-emptyfinal IndexedDISI disi =- new IndexedDISI(- data,+ newIndexedDISI(ordsEntry.docsWithFieldOffset,ordsEntry.docsWithFieldLength,ordsEntry.jumpTableEntryCount,@@ -2458,6 +2514,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {if (skipIndexData != null) {CodecUtil.checksumEntireFile(skipIndexData, merge);}+ if (disiData != null) {+ CodecUtil.checksumEntireFile(disiData, merge);+ }}/**diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.javaindex ad5f6108917..c335b193688 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java@@ -435,6 +435,68 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatTdir.close();}+ public void testSparseDISIStoredInSeparateFile() throws Exception {+ // A .dvp always gets a header + footer, so just checking it exists proves nothing. Instead+ // compare a sparse field against a dense-only baseline: only the sparse one should write an+ // IndexedDISI, so its .dvp must be bigger.+ final int numDocs = 512;++ // Dense baseline: no IndexedDISI, so .dvp is just header + footer.+ final long denseDvpBytes;+ try (Directory dir = newDirectory()) {+ IndexWriterConfig conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMergeScheduler(new SerialMergeScheduler())+ .setUseCompoundFile(false); // keep .dvp visible in the listing+ try (IndexWriter writer = new IndexWriter(dir, conf)) {+ for (int i = 0; i < numDocs; i++) {+ Document doc = new Document();+ doc.add(new NumericDocValuesField("dv", i)); // every doc -> dense+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ }+ denseDvpBytes = totalBytes(dir, Lucene90DocValuesFormat.DISI_EXTENSION);+ assertTrue("a segment with doc values should always have a .dvp", denseDvpBytes > 0);+ }++ // Sparse field: only some docs have a value, so we expect an IndexedDISI in .dvp.+ try (Directory dir = newDirectory()) {+ IndexWriterConfig conf =+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMergeScheduler(new SerialMergeScheduler())+ .setUseCompoundFile(false);+ try (IndexWriter writer = new IndexWriter(dir, conf)) {+ for (int i = 0; i < numDocs; i++) {+ Document doc = new Document();+ if (i % 2 == 0) { // only even docs -> sparse+ doc.add(new NumericDocValuesField("dv", i));+ }+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ }+ final long sparseDvpBytes = totalBytes(dir, Lucene90DocValuesFormat.DISI_EXTENSION);+ assertTrue(+ "sparse IndexedDISI should live in .dvp (sparse="+ + sparseDvpBytes+ + "B, dense="+ + denseDvpBytes+ + "B)",+ sparseDvpBytes > denseDvpBytes);+ }+ }++ private static long totalBytes(Directory dir, String extension) throws IOException {+ long total = 0;+ for (String file : dir.listAll()) {+ if (file.endsWith("." + extension)) {+ total += dir.fileLength(file);+ }+ }+ return total;+ }+private void doTestSparseDocValuesVsStoredFields() throws Exception {final long[] values = new long[TestUtil.nextInt(random(), 1, 500)];for (int i = 0; i < values.length; ++i) {