Skip to content

Apache Lucene open-source search software

Store DocValuesSkippers in a separate file (#15976)

Alan Woodward37b6d65a7d6d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7c17ef0d8e9..278a4ae28b7 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -299,6 +299,8 @@ Improvements
buffer was controlled by an explicit parameter (set as `resultSimilarity` - `traversalSimilarity`), which is now
deprecated. (Kaival Parikh)
+* GITHUB#15976: Store DocValuesSkipper data in their own file. (Alan Woodward)
+
Optimizations
---------------------
* GITHUB#15861: Optimise PhraseScorer by short circuiting non competitive documents in TOP_SCORES mode. (Prithvi S)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
index 8223320dfcc..833acb26a49 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
@@ -63,7 +63,7 @@ import org.apache.lucene.util.packed.DirectWriter;
/** writer for {@link Lucene90DocValuesFormat} */
final class Lucene90DocValuesConsumer extends DocValuesConsumer {
- IndexOutput data, meta;
+ IndexOutput data, meta, skipIndex;
final int maxDoc;
private byte[] termsDictBuffer;
private final int skipIndexIntervalSize;
@@ -75,7 +75,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
String dataCodec,
String dataExtension,
String metaCodec,
- String metaExtension)
+ String metaExtension,
+ String skipIndexCodec,
+ String skipIndexExtension)
throws IOException {
this.termsDictBuffer = new byte[1 << 14];
try {
@@ -99,6 +101,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
Lucene90DocValuesFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
+ String skipIndexName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, skipIndexExtension);
+ skipIndex = state.directory.createOutput(skipIndexName, state.context);
+ CodecUtil.writeIndexHeader(
+ skipIndex,
+ skipIndexCodec,
+ Lucene90DocValuesFormat.VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
maxDoc = state.segmentInfo.maxDoc();
this.skipIndexIntervalSize = skipIndexIntervalSize;
} catch (Throwable t) {
@@ -118,13 +130,16 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
if (data != null) {
CodecUtil.writeFooter(data); // write checksum
}
+ if (skipIndex != null) {
+ CodecUtil.writeFooter(skipIndex); // write checksum
+ }
} catch (Throwable t) {
- IOUtils.closeWhileSuppressingExceptions(t, data, meta);
+ IOUtils.closeWhileSuppressingExceptions(t, data, meta, skipIndex);
throw t;
}
- IOUtils.close(data, meta);
+ IOUtils.close(data, meta, skipIndex);
} finally {
- meta = data = null;
+ meta = data = skipIndex = null;
}
}
@@ -246,7 +261,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer)
throws IOException {
assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;
- final long start = data.getFilePointer();
+ final long start = skipIndex.getFilePointer();
final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
long globalMaxValue = Long.MIN_VALUE;
long globalMinValue = Long.MAX_VALUE;
@@ -288,7 +303,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
writeLevels(accumulators);
}
meta.writeLong(start); // record the start in meta
- meta.writeLong(data.getFilePointer() - start); // record the length
+ meta.writeLong(skipIndex.getFilePointer() - start); // record the length
assert globalDocCount == 0 || globalMaxValue >= globalMinValue;
meta.writeLong(globalMaxValue);
meta.writeLong(globalMinValue);
@@ -308,17 +323,17 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
// compute how many levels we need to write for the current accumulator
final int levels = getLevels(index, totalAccumulators);
// write the number of levels
- data.writeByte((byte) levels);
+ skipIndex.writeByte((byte) levels);
// write intervals in reverse order. This is done so we don't
// need to read all of them in case of slipping
for (int level = levels - 1; level >= 0; level--) {
final SkipAccumulator accumulator =
accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level));
- data.writeInt(accumulator.maxDocID);
- data.writeInt(accumulator.minDocID);
- data.writeLong(accumulator.maxValue);
- data.writeLong(accumulator.minValue);
- data.writeInt(accumulator.docCount);
+ skipIndex.writeInt(accumulator.maxDocID);
+ skipIndex.writeInt(accumulator.minDocID);
+ skipIndex.writeLong(accumulator.maxValue);
+ skipIndex.writeLong(accumulator.minValue);
+ skipIndex.writeInt(accumulator.docCount);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
index 922bc2481c9..cd4bb7ef076 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
@@ -132,6 +132,7 @@ import org.apache.lucene.util.packed.DirectWriter;
* <ol>
* <li><code>.dvd</code>: DocValues data
* <li><code>.dvm</code>: DocValues metadata
+ * <li><code>.dvs</code>: DocValues skip index (since version 1)
* </ol>
*
* @lucene.experimental
@@ -158,21 +159,37 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
return new Lucene90DocValuesConsumer(
- state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+ state,
+ skipIndexIntervalSize,
+ DATA_CODEC,
+ DATA_EXTENSION,
+ META_CODEC,
+ META_EXTENSION,
+ SKIP_INDEX_CODEC,
+ SKIP_INDEX_EXTENSION);
}
@Override
public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
return new Lucene90DocValuesProducer(
- state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+ state,
+ DATA_CODEC,
+ DATA_EXTENSION,
+ META_CODEC,
+ META_EXTENSION,
+ SKIP_INDEX_CODEC,
+ SKIP_INDEX_EXTENSION);
}
static final String DATA_CODEC = "Lucene90DocValuesData";
static final String DATA_EXTENSION = "dvd";
static final String META_CODEC = "Lucene90DocValuesMetadata";
static final String META_EXTENSION = "dvm";
+ static final String SKIP_INDEX_CODEC = "Lucene90DocValuesSkipIndex";
+ static final String SKIP_INDEX_EXTENSION = "dvs";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_SKIPPER_SEPARATE_FILE = 1;
+ static final int VERSION_CURRENT = VERSION_SKIPPER_SEPARATE_FILE;
// indicates docvalues type
static final byte NUMERIC = 0;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
index 71fccdbf825..c20baa4b35e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
@@ -66,6 +66,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<SortedNumericEntry> sortedNumerics;
private final IntObjectHashMap<DocValuesSkipperEntry> skippers;
private final IndexInput data;
+ private final IndexInput skipIndexData;
private final int maxDoc;
private int version = -1;
private final boolean merging;
@@ -76,7 +77,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
String dataCodec,
String dataExtension,
String metaCodec,
- String metaExtension)
+ String metaExtension,
+ String skipIndexCodec,
+ String skipIndexExtension)
throws IOException {
String metaName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
@@ -139,6 +142,35 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
IOUtils.closeWhileSuppressingExceptions(t, data);
throw t;
}
+
+ if (version >= Lucene90DocValuesFormat.VERSION_SKIPPER_SEPARATE_FILE) {
+ String skipIndexName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, skipIndexExtension);
+ this.skipIndexData =
+ state.directory.openInput(skipIndexName, state.context.withHints(FileTypeHint.INDEX));
+ try {
+ final int skipVersion =
+ CodecUtil.checkIndexHeader(
+ skipIndexData,
+ skipIndexCodec,
+ Lucene90DocValuesFormat.VERSION_SKIPPER_SEPARATE_FILE,
+ Lucene90DocValuesFormat.VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ if (version != skipVersion) {
+ throw new CorruptIndexException(
+ "Format versions mismatch: meta=" + version + ", skipIndex=" + skipVersion,
+ skipIndexData);
+ }
+ CodecUtil.retrieveChecksum(skipIndexData);
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, data, skipIndexData);
+ throw t;
+ }
+ } else {
+ this.skipIndexData = null;
+ }
}
// Used for cloning
@@ -150,6 +182,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
IntObjectHashMap<SortedNumericEntry> sortedNumerics,
IntObjectHashMap<DocValuesSkipperEntry> skippers,
IndexInput data,
+ IndexInput skipIndexData,
int maxDoc,
int version,
boolean merging) {
@@ -160,6 +193,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
this.sortedNumerics = sortedNumerics;
this.skippers = skippers;
this.data = data.clone();
+ this.skipIndexData = skipIndexData != null ? skipIndexData.clone() : null;
this.maxDoc = maxDoc;
this.version = version;
this.merging = merging;
@@ -175,6 +209,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
sortedNumerics,
skippers,
data,
+ skipIndexData,
maxDoc,
version,
true);
@@ -349,7 +384,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public void close() throws IOException {
- data.close();
+ IOUtils.close(data, skipIndexData);
}
private record DocValuesSkipperEntry(
@@ -1784,6 +1819,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public void checkIntegrity() throws IOException {
CodecUtil.checksumEntireFile(data);
+ if (skipIndexData != null) {
+ CodecUtil.checksumEntireFile(skipIndexData);
+ }
}
/**
@@ -1864,7 +1902,8 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
final DocValuesSkipperEntry entry = skippers.get(field.number);
- final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);
+ final IndexInput skipperSource = skipIndexData != null ? skipIndexData : data;
+ final IndexInput input = skipperSource.slice("doc value skipper", entry.offset, entry.length);
// TODO: should we write to disk the actual max level for this segment?
return new DocValuesSkipper() {
final int[] minDocID = new int[SKIP_INDEX_MAX_LEVEL];
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
index 14f2a541703..50cb8b7aab3 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
@@ -45,6 +45,7 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -64,6 +65,10 @@ import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.ByteBuffersDataInput;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
import org.apache.lucene.tests.index.BaseCompressingDocValuesFormatTestCase;
@@ -1020,4 +1025,52 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatT
reader.close();
directory.close();
}
+
+ public void testSkipIndexStoredSeparately() throws IOException {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig conf = newIndexWriterConfig();
+ conf.setCodec(getCodec());
+ conf.setUseCompoundFile(false);
+ try (IndexWriter writer = new IndexWriter(dir, conf)) {
+ for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("dv", i));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+
+ // Wrap the directory so that any attempt to slice() the .dvd file throws.
+ // The producer constructor opens .dvd for header/checksum validation (sequential reads),
+ // but getSkipper() should only slice the .dvs file, never the .dvd file.
+ Directory noDataSliceDir =
+ new FilterDirectory(dir) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ if (name.endsWith("." + Lucene90DocValuesFormat.DATA_EXTENSION)) {
+ return new FilterIndexInput("no-slice-" + name, in) {
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) {
+ throw new AssertionError(
+ "should not slice .dvd file for skip index access: " + sliceDescription);
+ }
+ };
+ }
+ return in;
+ }
+ };
+
+ try (DirectoryReader reader = DirectoryReader.open(noDataSliceDir)) {
+ LeafReader leaf = getOnlyLeafReader(reader);
+ DocValuesSkipper skipper = leaf.getDocValuesSkipper("dv");
+ assertNotNull(skipper);
+ assertEquals(100, skipper.docCount());
+ skipper.advance(0);
+ assertEquals(0, skipper.minDocID(0));
+ assertTrue(skipper.maxDocID(0) >= 0);
+ assertTrue(skipper.minValue() <= skipper.maxValue());
+ }
+ }
+ }
}