Skip to content

Apache Lucene open-source search software

Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)

Adrien Grandc0fd4404ea3a
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
index 6497bb13d11..4360b90f237 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
@@ -75,7 +75,11 @@ public final class Lucene90RWPostingsFormat extends PostingsFormat {
try {
FieldsConsumer ret =
new Lucene90BlockTreeTermsWriter(
- state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+ state,
+ postingsWriter,
+ minTermBlockSize,
+ maxTermBlockSize,
+ Lucene90BlockTreeTermsReader.VERSION_START);
success = true;
return ret;
} finally {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 463532218cd..26aea1dda59 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -238,6 +238,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
final int maxDoc;
final int minItemsInBlock;
final int maxItemsInBlock;
+ final int version;
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
@@ -255,10 +256,37 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
int minItemsInBlock,
int maxItemsInBlock)
throws IOException {
+ this(
+ state,
+ postingsWriter,
+ minItemsInBlock,
+ maxItemsInBlock,
+ Lucene90BlockTreeTermsReader.VERSION_CURRENT);
+ }
+
+ /** Expert constructor that allows configuring the version, used for bw tests. */
+ public Lucene90BlockTreeTermsWriter(
+ SegmentWriteState state,
+ PostingsWriterBase postingsWriter,
+ int minItemsInBlock,
+ int maxItemsInBlock,
+ int version)
+ throws IOException {
validateSettings(minItemsInBlock, maxItemsInBlock);
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
+ if (version < Lucene90BlockTreeTermsReader.VERSION_START
+ || version > Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+ throw new IllegalArgumentException(
+ "Expected version in range ["
+ + Lucene90BlockTreeTermsReader.VERSION_START
+ + ", "
+ + Lucene90BlockTreeTermsReader.VERSION_CURRENT
+ + "], but got "
+ + version);
+ }
+ this.version = version;
this.maxDoc = state.segmentInfo.maxDoc();
this.fieldInfos = state.fieldInfos;
@@ -276,7 +304,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
CodecUtil.writeIndexHeader(
termsOut,
Lucene90BlockTreeTermsReader.TERMS_CODEC_NAME,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+ version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -289,7 +317,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
CodecUtil.writeIndexHeader(
indexOut,
Lucene90BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+ version,
state.segmentInfo.getId(),
state.segmentSuffix);
// segment = state.segmentInfo.name;
@@ -303,7 +331,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
CodecUtil.writeIndexHeader(
metaOut,
Lucene90BlockTreeTermsReader.TERMS_META_CODEC_NAME,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+ version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -451,7 +479,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL)));
}
- private static final class PendingBlock extends PendingEntry {
+ private final class PendingBlock extends PendingEntry {
public final BytesRef prefix;
public final long fp;
public FST<BytesRef> index;
@@ -494,7 +522,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
assert scratchBytes.size() == 0;
// write the leading vLong in MSB order for better outputs sharing in the FST
- writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
+ if (version >= Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT) {
+ writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
+ } else {
+ scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
+ }
if (isFloor) {
scratchBytes.writeVInt(blocks.size() - 1);
for (int i = 1; i < blocks.size(); i++) {
@@ -522,12 +554,19 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));
final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+ final int fstVersion;
+ if (version >= Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+ fstVersion = FST.VERSION_CURRENT;
+ } else {
+ fstVersion = FST.VERSION_90;
+ }
final FSTCompiler<BytesRef> fstCompiler =
new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
// Disable suffixes sharing for block tree index because suffixes are mostly dropped
// from the FST index and left in the term blocks.
.suffixRAMLimitMB(0d)
.dataOutput(getOnHeapReaderWriter(pageBits))
+ .setVersion(fstVersion)
.build();
// if (DEBUG) {
// System.out.println(" compile index for prefix=" + prefix);
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index f49f61767e4..cc161ba3f8d 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -109,10 +109,23 @@ public final class FST<T> implements Accountable {
// Increment version to change it
private static final String FILE_FORMAT_NAME = "FST";
- private static final int VERSION_START = 6;
+
+ /** First supported version, this is the version that was used when releasing Lucene 7.0. */
+ public static final int VERSION_START = 6;
+
+ // Version 7 introduced direct addressing for arcs, but it's not recorded here because it doesn't
+ // need version checks on the read side, it uses new flag values on arcs instead.
+
private static final int VERSION_LITTLE_ENDIAN = 8;
- private static final int VERSION_CONTINUOUS_ARCS = 9;
- static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;
+
+ /** Version that started storing continuous arcs. */
+ public static final int VERSION_CONTINUOUS_ARCS = 9;
+
+ /** Current version. */
+ public static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;
+
+ /** Version that was used when releasing Lucene 9.0. */
+ public static final int VERSION_90 = VERSION_LITTLE_ENDIAN;
// Never serialized; just used to represent the virtual
// final node w/ no arcs:
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
index da1fff8ec15..961511616ab 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -29,7 +29,6 @@ import static org.apache.lucene.util.fst.FST.BIT_STOP_NODE;
import static org.apache.lucene.util.fst.FST.BIT_TARGET_NEXT;
import static org.apache.lucene.util.fst.FST.FINAL_END_NODE;
import static org.apache.lucene.util.fst.FST.NON_FINAL_END_NODE;
-import static org.apache.lucene.util.fst.FST.VERSION_CURRENT;
import static org.apache.lucene.util.fst.FST.getNumPresenceBytes;
import java.io.IOException;
@@ -135,6 +134,7 @@ public class FSTCompiler<T> {
final boolean allowFixedLengthArcs;
final float directAddressingMaxOversizingFactor;
+ final int version;
long directAddressingExpansionCredit;
// the DataOutput to stream the FST bytes to
@@ -163,10 +163,12 @@ public class FSTCompiler<T> {
Outputs<T> outputs,
boolean allowFixedLengthArcs,
DataOutput dataOutput,
- float directAddressingMaxOversizingFactor)
+ float directAddressingMaxOversizingFactor,
+ int version)
throws IOException {
this.allowFixedLengthArcs = allowFixedLengthArcs;
this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor;
+ this.version = version;
// pad: ensure no node gets address 0 which is reserved to mean
// the stop state w/ no arcs
dataOutput.writeByte((byte) 0);
@@ -174,7 +176,7 @@ public class FSTCompiler<T> {
this.dataOutput = dataOutput;
fst =
new FST<>(
- new FST.FSTMetadata<>(inputType, outputs, null, -1, VERSION_CURRENT, 0),
+ new FST.FSTMetadata<>(inputType, outputs, null, -1, version, 0),
toFSTReader(dataOutput));
if (suffixRAMLimitMB < 0) {
throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
@@ -241,6 +243,7 @@ public class FSTCompiler<T> {
private boolean allowFixedLengthArcs = true;
private DataOutput dataOutput;
private float directAddressingMaxOversizingFactor = DIRECT_ADDRESSING_MAX_OVERSIZING_FACTOR;
+ private int version = FST.VERSION_CURRENT;
/**
* @param inputType The input type (transition labels). Can be anything from {@link INPUT_TYPE}
@@ -325,6 +328,21 @@ public class FSTCompiler<T> {
return this;
}
+ /** Expert: Set the codec version. * */
+ public Builder<T> setVersion(int version) {
+ if (version < FST.VERSION_90 || version > FST.VERSION_CURRENT) {
+ throw new IllegalArgumentException(
+ "Expected version in range ["
+ + FST.VERSION_90
+ + ", "
+ + FST.VERSION_CURRENT
+ + "], got "
+ + version);
+ }
+ this.version = version;
+ return this;
+ }
+
/** Creates a new {@link FSTCompiler}. */
public FSTCompiler<T> build() throws IOException {
// create a default DataOutput if not specified
@@ -337,7 +355,8 @@ public class FSTCompiler<T> {
outputs,
allowFixedLengthArcs,
dataOutput,
- directAddressingMaxOversizingFactor);
+ directAddressingMaxOversizingFactor,
+ version);
}
}
@@ -517,7 +536,7 @@ public class FSTCompiler<T> {
int labelRange = nodeIn.arcs[nodeIn.numArcs - 1].label - nodeIn.arcs[0].label + 1;
assert labelRange > 0;
boolean continuousLabel = labelRange == nodeIn.numArcs;
- if (continuousLabel) {
+ if (continuousLabel && version >= FST.VERSION_CONTINUOUS_ARCS) {
writeNodeForDirectAddressingOrContinuous(
nodeIn, maxBytesPerArcWithoutLabel, labelRange, true);
continuousNodeCount++;