Apache Lucene open-source search software
Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)
Adrien Grandc0fd4404ea3a
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.javaindex 6497bb13d11..4360b90f237 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java@@ -75,7 +75,11 @@ public final class Lucene90RWPostingsFormat extends PostingsFormat {try {FieldsConsumer ret =new Lucene90BlockTreeTermsWriter(- state, postingsWriter, minTermBlockSize, maxTermBlockSize);+ state,+ postingsWriter,+ minTermBlockSize,+ maxTermBlockSize,+ Lucene90BlockTreeTermsReader.VERSION_START);success = true;return ret;} finally {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.javaindex 463532218cd..26aea1dda59 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java@@ -238,6 +238,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {final int maxDoc;final int minItemsInBlock;final int maxItemsInBlock;+ final int version;final PostingsWriterBase postingsWriter;final FieldInfos fieldInfos;@@ -255,10 +256,37 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {int minItemsInBlock,int maxItemsInBlock)throws IOException {+ this(+ state,+ postingsWriter,+ minItemsInBlock,+ maxItemsInBlock,+ Lucene90BlockTreeTermsReader.VERSION_CURRENT);+ }++ /** Expert constructor that allows configuring the version, used for bw tests. */+ public Lucene90BlockTreeTermsWriter(+ SegmentWriteState state,+ PostingsWriterBase postingsWriter,+ int minItemsInBlock,+ int maxItemsInBlock,+ int version)+ throws IOException {validateSettings(minItemsInBlock, maxItemsInBlock);this.minItemsInBlock = minItemsInBlock;this.maxItemsInBlock = maxItemsInBlock;+ if (version < Lucene90BlockTreeTermsReader.VERSION_START+ || version > Lucene90BlockTreeTermsReader.VERSION_CURRENT) {+ throw new IllegalArgumentException(+ "Expected version in range ["+ + Lucene90BlockTreeTermsReader.VERSION_START+ + ", "+ + Lucene90BlockTreeTermsReader.VERSION_CURRENT+ + "], but got "+ + version);+ }+ this.version = version;this.maxDoc = state.segmentInfo.maxDoc();this.fieldInfos = state.fieldInfos;@@ -276,7 +304,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {CodecUtil.writeIndexHeader(termsOut,Lucene90BlockTreeTermsReader.TERMS_CODEC_NAME,- Lucene90BlockTreeTermsReader.VERSION_CURRENT,+ version,state.segmentInfo.getId(),state.segmentSuffix);@@ -289,7 +317,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {CodecUtil.writeIndexHeader(indexOut,Lucene90BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,- Lucene90BlockTreeTermsReader.VERSION_CURRENT,+ version,state.segmentInfo.getId(),state.segmentSuffix);// segment = state.segmentInfo.name;@@ -303,7 +331,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {CodecUtil.writeIndexHeader(metaOut,Lucene90BlockTreeTermsReader.TERMS_META_CODEC_NAME,- Lucene90BlockTreeTermsReader.VERSION_CURRENT,+ version,state.segmentInfo.getId(),state.segmentSuffix);@@ -451,7 +479,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL)));}- private static final class PendingBlock extends PendingEntry {+ private final class PendingBlock extends PendingEntry {public final BytesRef prefix;public final long fp;public FST<BytesRef> index;@@ -494,7 +522,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {assert scratchBytes.size() == 0;// write the leading vLong in MSB order for better outputs sharing in the FST- writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);+ if (version >= Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT) {+ writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);+ } else {+ scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));+ }if (isFloor) {scratchBytes.writeVInt(blocks.size() - 1);for (int i = 1; i < blocks.size(); i++) {@@ -522,12 +554,19 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();+ final int fstVersion;+ if (version >= Lucene90BlockTreeTermsReader.VERSION_CURRENT) {+ fstVersion = FST.VERSION_CURRENT;+ } else {+ fstVersion = FST.VERSION_90;+ }final FSTCompiler<BytesRef> fstCompiler =new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)// Disable suffixes sharing for block tree index because suffixes are mostly dropped// from the FST index and left in the term blocks..suffixRAMLimitMB(0d).dataOutput(getOnHeapReaderWriter(pageBits))+ .setVersion(fstVersion).build();// if (DEBUG) {// System.out.println(" compile index for prefix=" + prefix);diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.javaindex f49f61767e4..cc161ba3f8d 100644--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java@@ -109,10 +109,23 @@ public final class FST<T> implements Accountable {// Increment version to change itprivate static final String FILE_FORMAT_NAME = "FST";- private static final int VERSION_START = 6;++ /** First supported version, this is the version that was used when releasing Lucene 7.0. */+ public static final int VERSION_START = 6;++ // Version 7 introduced direct addressing for arcs, but it's not recorded here because it doesn't+ // need version checks on the read side, it uses new flag values on arcs instead.+private static final int VERSION_LITTLE_ENDIAN = 8;- private static final int VERSION_CONTINUOUS_ARCS = 9;- static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;++ /** Version that started storing continuous arcs. */+ public static final int VERSION_CONTINUOUS_ARCS = 9;++ /** Current version. */+ public static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;++ /** Version that was used when releasing Lucene 9.0. */+ public static final int VERSION_90 = VERSION_LITTLE_ENDIAN;// Never serialized; just used to represent the virtual// final node w/ no arcs:diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.javaindex da1fff8ec15..961511616ab 100644--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java@@ -29,7 +29,6 @@ import static org.apache.lucene.util.fst.FST.BIT_STOP_NODE;import static org.apache.lucene.util.fst.FST.BIT_TARGET_NEXT;import static org.apache.lucene.util.fst.FST.FINAL_END_NODE;import static org.apache.lucene.util.fst.FST.NON_FINAL_END_NODE;-import static org.apache.lucene.util.fst.FST.VERSION_CURRENT;import static org.apache.lucene.util.fst.FST.getNumPresenceBytes;import java.io.IOException;@@ -135,6 +134,7 @@ public class FSTCompiler<T> {final boolean allowFixedLengthArcs;final float directAddressingMaxOversizingFactor;+ final int version;long directAddressingExpansionCredit;// the DataOutput to stream the FST bytes to@@ -163,10 +163,12 @@ public class FSTCompiler<T> {Outputs<T> outputs,boolean allowFixedLengthArcs,DataOutput dataOutput,- float directAddressingMaxOversizingFactor)+ float directAddressingMaxOversizingFactor,+ int version)throws IOException {this.allowFixedLengthArcs = allowFixedLengthArcs;this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor;+ this.version = version;// pad: ensure no node gets address 0 which is reserved to mean// the stop state w/ no arcsdataOutput.writeByte((byte) 0);@@ -174,7 +176,7 @@ public class FSTCompiler<T> {this.dataOutput = dataOutput;fst =new FST<>(- new FST.FSTMetadata<>(inputType, outputs, null, -1, VERSION_CURRENT, 0),+ new FST.FSTMetadata<>(inputType, outputs, null, -1, version, 0),toFSTReader(dataOutput));if (suffixRAMLimitMB < 0) {throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);@@ -241,6 +243,7 @@ public class FSTCompiler<T> {private boolean allowFixedLengthArcs = true;private DataOutput dataOutput;private float directAddressingMaxOversizingFactor = DIRECT_ADDRESSING_MAX_OVERSIZING_FACTOR;+ private int version = FST.VERSION_CURRENT;/*** @param inputType The input type (transition labels). Can be anything from {@link INPUT_TYPE}@@ -325,6 +328,21 @@ public class FSTCompiler<T> {return this;}+ /** Expert: Set the codec version. * */+ public Builder<T> setVersion(int version) {+ if (version < FST.VERSION_90 || version > FST.VERSION_CURRENT) {+ throw new IllegalArgumentException(+ "Expected version in range ["+ + FST.VERSION_90+ + ", "+ + FST.VERSION_CURRENT+ + "], got "+ + version);+ }+ this.version = version;+ return this;+ }+/** Creates a new {@link FSTCompiler}. */public FSTCompiler<T> build() throws IOException {// create a default DataOutput if not specified@@ -337,7 +355,8 @@ public class FSTCompiler<T> {outputs,allowFixedLengthArcs,dataOutput,- directAddressingMaxOversizingFactor);+ directAddressingMaxOversizingFactor,+ version);}}@@ -517,7 +536,7 @@ public class FSTCompiler<T> {int labelRange = nodeIn.arcs[nodeIn.numArcs - 1].label - nodeIn.arcs[0].label + 1;assert labelRange > 0;boolean continuousLabel = labelRange == nodeIn.numArcs;- if (continuousLabel) {+ if (continuousLabel && version >= FST.VERSION_CONTINUOUS_ARCS) {writeNodeForDirectAddressingOrContinuous(nodeIn, maxBytesPerArcWithoutLabel, labelRange, true);continuousNodeCount++;