Skip to content

Apache Lucene open-source search software

Store doc values disi data into a separate file (#16540)

Sagar Upadhyaya1160d3c8a256
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5944d5453a6..2cdc8ebdf2e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -404,6 +404,8 @@ New Features
Improvements
---------------------
+* GITHUB#16298: Store doc values DISI data into a separate file. (Sagar Upadhyaya)
+
* GITHUB#16247: Introduce CachingCollectorManager to parallelize search when using CachingCollector.
Remove experimental GroupingSearch constructor that takes a GroupSelector as argument
in favour of providing a GroupSelector supplier. (Binlong Gao)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
index a7dff6b9e2c..5bdd5d9fba9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
@@ -64,7 +64,7 @@ import org.apache.lucene.util.packed.DirectWriter;
/** writer for {@link Lucene90DocValuesFormat} */
final class Lucene90DocValuesConsumer extends DocValuesConsumer {
- IndexOutput data, meta, skipIndex;
+ IndexOutput data, meta, skipIndex, disiData;
final int maxDoc;
private byte[] termsDictBuffer;
private final int skipIndexIntervalSize;
@@ -78,7 +78,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
String metaCodec,
String metaExtension,
String skipIndexCodec,
- String skipIndexExtension)
+ String skipIndexExtension,
+ String disiCodec,
+ String disiExtension)
throws IOException {
this.termsDictBuffer = new byte[1 << 14];
try {
@@ -112,6 +114,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
Lucene90DocValuesFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
+ String disiName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, disiExtension);
+ disiData = state.directory.createOutput(disiName, state.context);
+ CodecUtil.writeIndexHeader(
+ disiData,
+ disiCodec,
+ Lucene90DocValuesFormat.VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
maxDoc = state.segmentInfo.maxDoc();
this.skipIndexIntervalSize = skipIndexIntervalSize;
} catch (Throwable t) {
@@ -134,16 +146,33 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
if (skipIndex != null) {
CodecUtil.writeFooter(skipIndex); // write checksum
}
+ if (disiData != null) {
+ CodecUtil.writeFooter(disiData); // write checksum
+ }
} catch (Throwable t) {
- IOUtils.closeWhileSuppressingExceptions(t, data, meta, skipIndex);
+ IOUtils.closeWhileSuppressingExceptions(t, data, meta, skipIndex, disiData);
throw t;
}
- IOUtils.close(data, meta, skipIndex);
+ IOUtils.close(data, meta, skipIndex, disiData);
} finally {
- meta = data = skipIndex = null;
+ meta = data = skipIndex = disiData = null;
}
}
+ /**
+ * Writes a sparse field's {@link IndexedDISI} to the {@code .dvp} file and records its offset,
+ * length and shape in the metadata.
+ */
+ private void writeDISI(DocIdSetIterator values) throws IOException {
+ final long offset = disiData.getFilePointer();
+ meta.writeLong(offset); // docsWithFieldOffset (into .dvp)
+ final short jumpTableEntryCount =
+ IndexedDISI.writeBitSet(values, disiData, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+ meta.writeLong(disiData.getFilePointer() - offset); // docsWithFieldLength (region in .dvp)
+ meta.writeShort(jumpTableEntryCount);
+ meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+ }
+
@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer)
throws IOException {
@@ -440,15 +469,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
meta.writeLong(0L); // docsWithFieldLength
meta.writeShort((short) -1); // jumpTableEntryCount
meta.writeByte((byte) -1); // denseRankPower
- } else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values
- long offset = data.getFilePointer();
- meta.writeLong(offset); // docsWithFieldOffset
+ } else { // meta[disi.offset, disi.length]: IndexedDISI region in .dvp for documents with values
values = valuesProducer.getSortedNumeric(field);
- final short jumpTableEntryCount =
- IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
- meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
- meta.writeShort(jumpTableEntryCount);
- meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+ writeDISI(values);
}
meta.writeLong(numValues);
@@ -633,14 +656,8 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
meta.writeShort((short) -1); // jumpTableEntryCount
meta.writeByte((byte) -1); // denseRankPower
} else {
- long offset = data.getFilePointer();
- meta.writeLong(offset); // docsWithFieldOffset
values = valuesProducer.getBinary(field);
- final short jumpTableEntryCount =
- IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
- meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
- meta.writeShort(jumpTableEntryCount);
- meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+ writeDISI(values);
}
meta.writeInt(numDocsWithField);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
index 22cd7581f1d..777e76d9792 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
@@ -133,6 +133,8 @@ import org.apache.lucene.util.packed.DirectWriter;
* <li><code>.dvd</code>: DocValues data
* <li><code>.dvm</code>: DocValues metadata
* <li><code>.dvs</code>: DocValues skip index (since version 1)
+ * <li><code>.dvp</code>: DocValues presence, i.e. the {@link IndexedDISI} for sparse fields
+ * (since version 3)
* </ol>
*
* @lucene.experimental
@@ -166,7 +168,9 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
META_CODEC,
META_EXTENSION,
SKIP_INDEX_CODEC,
- SKIP_INDEX_EXTENSION);
+ SKIP_INDEX_EXTENSION,
+ DISI_CODEC,
+ DISI_EXTENSION);
}
@Override
@@ -178,7 +182,9 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
META_CODEC,
META_EXTENSION,
SKIP_INDEX_CODEC,
- SKIP_INDEX_EXTENSION);
+ SKIP_INDEX_EXTENSION,
+ DISI_CODEC,
+ DISI_EXTENSION);
}
static final String DATA_CODEC = "Lucene90DocValuesData";
@@ -187,10 +193,14 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
static final String META_EXTENSION = "dvm";
static final String SKIP_INDEX_CODEC = "Lucene90DocValuesSkipIndex";
static final String SKIP_INDEX_EXTENSION = "dvs";
+ static final String DISI_CODEC = "Lucene90DocValuesDISI";
+ static final String DISI_EXTENSION = "dvp";
static final int VERSION_START = 0;
static final int VERSION_SKIPPER_SEPARATE_FILE = 1;
static final int VERSION_SKIPPER_MAX_VALUE_COUNT = 2;
- static final int VERSION_CURRENT = VERSION_SKIPPER_MAX_VALUE_COUNT;
+ // Moves the sparse-field IndexedDISI from .dvd into a dedicated .dvp file.
+ static final int VERSION_DISI_SEPARATE_FILE = 3;
+ static final int VERSION_CURRENT = VERSION_DISI_SEPARATE_FILE;
// indicates docvalues type
static final byte NUMERIC = 0;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index cdefbca58ff..dd67e555e5e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -69,6 +69,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<DocValuesSkipperEntry> skippers;
private final IndexInput data;
private final IndexInput skipIndexData;
+ // Sparse-field presence (IndexedDISI) file. Null for pre-VERSION_DISI_SEPARATE_FILE segments,
+ // which still keep the IndexedDISI in .dvd.
+ private final IndexInput disiData;
private final int maxDoc;
private int version = -1;
private final boolean merging;
@@ -81,7 +84,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
String metaCodec,
String metaExtension,
String skipIndexCodec,
- String skipIndexExtension)
+ String skipIndexExtension,
+ String disiCodec,
+ String disiExtension)
throws IOException {
String metaName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
@@ -178,6 +183,35 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
} else {
this.skipIndexData = null;
}
+
+ if (version >= Lucene90DocValuesFormat.VERSION_DISI_SEPARATE_FILE) {
+ IndexInput disiIn = null;
+ try {
+ String disiName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, disiExtension);
+ disiIn = state.directory.openInput(disiName, state.context.withHints(FileTypeHint.INDEX));
+ final int disiVersion =
+ CodecUtil.checkIndexHeader(
+ disiIn,
+ disiCodec,
+ Lucene90DocValuesFormat.VERSION_DISI_SEPARATE_FILE,
+ Lucene90DocValuesFormat.VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ if (version != disiVersion) {
+ throw new CorruptIndexException(
+ "Format versions mismatch: meta=" + version + ", disi=" + disiVersion, disiIn);
+ }
+ CodecUtil.retrieveChecksum(disiIn);
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, data, skipIndexData, disiIn);
+ throw t;
+ }
+ this.disiData = disiIn;
+ } else {
+ this.disiData = null;
+ }
}
// Used for cloning
@@ -190,6 +224,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
IntObjectHashMap<DocValuesSkipperEntry> skippers,
IndexInput data,
IndexInput skipIndexData,
+ IndexInput disiData,
int maxDoc,
int version,
boolean merging) {
@@ -201,6 +236,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
this.skippers = skippers;
this.data = data.clone();
this.skipIndexData = skipIndexData != null ? skipIndexData.clone() : null;
+ this.disiData = disiData != null ? disiData.clone() : null;
this.maxDoc = maxDoc;
this.version = version;
this.merging = merging;
@@ -217,11 +253,36 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
skippers,
data,
skipIndexData,
+ disiData,
maxDoc,
version,
true);
}
+ /**
+ * Builds a sparse field's {@link IndexedDISI}.
+ *
+ * <p>Since {@link Lucene90DocValuesFormat#VERSION_DISI_SEPARATE_FILE} the bytes live in the
+ * {@code .dvp} file; for older segments they are inline in {@code .dvd}. The offset, length and
+ * shape are read from the metadata in both cases.
+ */
+ private IndexedDISI newIndexedDISI(
+ long docsWithFieldOffset,
+ long docsWithFieldLength,
+ short jumpTableEntryCount,
+ byte denseRankPower,
+ long numValues)
+ throws IOException {
+ final IndexInput in = disiData != null ? disiData : data;
+ return new IndexedDISI(
+ in,
+ docsWithFieldOffset,
+ docsWithFieldLength,
+ jumpTableEntryCount,
+ denseRankPower,
+ numValues);
+ }
+
private void inferMaxValueCounts(FieldInfos fieldInfos) {
for (var cursor : skippers) {
DocValuesSkipperEntry entry = cursor.value;
@@ -447,7 +508,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public void close() throws IOException {
- IOUtils.close(data, skipIndexData);
+ IOUtils.close(data, skipIndexData, disiData);
}
private record DocValuesSkipperEntry(
@@ -1020,8 +1081,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
} else {
// sparse
final IndexedDISI disi =
- new IndexedDISI(
- data,
+ newIndexedDISI(
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
@@ -1359,8 +1419,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
} else {
// sparse
final IndexedDISI disi =
- new IndexedDISI(
- data,
+ newIndexedDISI(
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
@@ -1497,8 +1556,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
};
} else if (ordsEntry.docsWithFieldOffset >= 0) { // sparse but non-empty
final IndexedDISI disi =
- new IndexedDISI(
- data,
+ newIndexedDISI(
ordsEntry.docsWithFieldOffset,
ordsEntry.docsWithFieldLength,
ordsEntry.jumpTableEntryCount,
@@ -2087,8 +2145,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
} else {
// sparse
final IndexedDISI disi =
- new IndexedDISI(
- data,
+ newIndexedDISI(
entry.docsWithFieldOffset,
entry.docsWithFieldLength,
entry.jumpTableEntryCount,
@@ -2324,8 +2381,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
};
} else if (ordsEntry.docsWithFieldOffset >= 0) { // sparse but non-empty
final IndexedDISI disi =
- new IndexedDISI(
- data,
+ newIndexedDISI(
ordsEntry.docsWithFieldOffset,
ordsEntry.docsWithFieldLength,
ordsEntry.jumpTableEntryCount,
@@ -2458,6 +2514,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
if (skipIndexData != null) {
CodecUtil.checksumEntireFile(skipIndexData, merge);
}
+ if (disiData != null) {
+ CodecUtil.checksumEntireFile(disiData, merge);
+ }
}
/**
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
index ad5f6108917..c335b193688 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
@@ -435,6 +435,68 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatT
dir.close();
}
+ public void testSparseDISIStoredInSeparateFile() throws Exception {
+ // A .dvp always gets a header + footer, so just checking it exists proves nothing. Instead
+ // compare a sparse field against a dense-only baseline: only the sparse one should write an
+ // IndexedDISI, so its .dvp must be bigger.
+ final int numDocs = 512;
+
+ // Dense baseline: no IndexedDISI, so .dvp is just header + footer.
+ final long denseDvpBytes;
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig conf =
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMergeScheduler(new SerialMergeScheduler())
+ .setUseCompoundFile(false); // keep .dvp visible in the listing
+ try (IndexWriter writer = new IndexWriter(dir, conf)) {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv", i)); // every doc -> dense
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+ denseDvpBytes = totalBytes(dir, Lucene90DocValuesFormat.DISI_EXTENSION);
+ assertTrue("a segment with doc values should always have a .dvp", denseDvpBytes > 0);
+ }
+
+ // Sparse field: only some docs have a value, so we expect an IndexedDISI in .dvp.
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig conf =
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMergeScheduler(new SerialMergeScheduler())
+ .setUseCompoundFile(false);
+ try (IndexWriter writer = new IndexWriter(dir, conf)) {
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ if (i % 2 == 0) { // only even docs -> sparse
+ doc.add(new NumericDocValuesField("dv", i));
+ }
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+ final long sparseDvpBytes = totalBytes(dir, Lucene90DocValuesFormat.DISI_EXTENSION);
+ assertTrue(
+ "sparse IndexedDISI should live in .dvp (sparse="
+ + sparseDvpBytes
+ + "B, dense="
+ + denseDvpBytes
+ + "B)",
+ sparseDvpBytes > denseDvpBytes);
+ }
+ }
+
+ private static long totalBytes(Directory dir, String extension) throws IOException {
+ long total = 0;
+ for (String file : dir.listAll()) {
+ if (file.endsWith("." + extension)) {
+ total += dir.fileLength(file);
+ }
+ }
+ return total;
+ }
+
private void doTestSparseDocValuesVsStoredFields() throws Exception {
final long[] values = new long[TestUtil.nextInt(random(), 1, 500)];
for (int i = 0; i < values.length; ++i) {