Apache Lucene open-source search software
Write MSB VLong for better outputs sharing in block tree index (#12631)
gf21214f01de2a2d03
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 736b7f2f4ef..0552300acae 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -178,6 +178,9 @@ Optimizations* GITHUB#12623: Use a MergeSorter taking advantage of extra storage for StableMSBRadixSorter. (Guo Feng)+* GITHUB#12623: Write MSB VLong for better outputs sharing in block tree index, decreasing ~14% size+ of .tip file. (Guo Feng)+Changes in runtime behavior---------------------diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.javaindex 7aba78112e8..39caaeb622d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java@@ -16,12 +16,15 @@*/package org.apache.lucene.codecs.lucene90.blocktree;+import static org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT;+import java.io.IOException;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.DataInput;import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.automaton.CompiledAutomaton;@@ -82,7 +85,7 @@ public final class FieldReader extends Terms {// + rootCode + " divisor=" + indexDivisor);// }rootBlockFP =- (new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length)).readVLong()+ readVLongOutput(new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length))>>> Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;// Initialize FST always off-heap.final IndexInput clone = indexIn.clone();@@ -99,6 +102,32 @@ public final class FieldReader extends Terms {*/}+ long readVLongOutput(DataInput in) throws IOException {+ if (parent.version >= VERSION_MSB_VLONG_OUTPUT) {+ return readMSBVLong(in);+ } else {+ return in.readVLong();+ }+ }++ /**+ * Decodes a variable length byte[] in MSB order back to long, as written by {@link+ * Lucene90BlockTreeTermsWriter#writeMSBVLong}.+ *+ * <p>Package private for testing.+ */+ static long readMSBVLong(DataInput in) throws IOException {+ long l = 0L;+ while (true) {+ byte b = in.readByte();+ l = (l << 7) | (b & 0x7FL);+ if ((b & 0x80) == 0) {+ break;+ }+ }+ return l;+ }+@Overridepublic BytesRef getMin() throws IOException {if (minTerm == null) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.javaindex eb60d7f3524..d9ca7a9bbd8 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java@@ -146,7 +146,7 @@ final class IntersectTermsEnumFrame {floorDataReader.reset(frameIndexData.bytes, frameIndexData.offset, frameIndexData.length);// Skip first long -- has redundant fp, hasTerms// flag, isFloor flag- final long code = floorDataReader.readVLong();+ final long code = ite.fr.readVLongOutput(floorDataReader);if ((code & Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0) {// Floor framenumFollowFloorBlocks = floorDataReader.readVInt();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.javaindex 4054e7a719c..1b385f8fa09 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java@@ -81,8 +81,13 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {/** Initial terms format. */public static final int VERSION_START = 0;+ /**+ * Version that encode output as MSB VLong for better outputs sharing in FST, see GITHUB#12620.+ */+ public static final int VERSION_MSB_VLONG_OUTPUT = 1;+/** Current terms format. */- public static final int VERSION_CURRENT = VERSION_START;+ public static final int VERSION_CURRENT = VERSION_MSB_VLONG_OUTPUT;/** Extension of terms index file */static final String TERMS_INDEX_EXTENSION = "tip";diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.javaindex 1b0bd3568c9..d0114ce0e79 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java@@ -430,6 +430,25 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {return brToString(new BytesRef(b));}+ /**+ * Encodes long value to variable length byte[], in MSB order. Use {@link+ * FieldReader#readMSBVLong} to decode.+ *+ * <p>Package private for testing+ */+ static void writeMSBVLong(long l, DataOutput scratchBytes) throws IOException {+ assert l >= 0;+ // Keep zero bits on most significant byte to have more chance to get prefix bytes shared.+ // e.g. we expect 0x7FFF stored as [0x81, 0xFF, 0x7F] but not [0xFF, 0xFF, 0x40]+ final int bytesNeeded = (Long.SIZE - Long.numberOfLeadingZeros(l) - 1) / 7 + 1;+ l <<= Long.SIZE - bytesNeeded * 7;+ for (int i = 1; i < bytesNeeded; i++) {+ scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL) | 0x80));+ l = l << 7;+ }+ scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL)));+ }+private static final class PendingBlock extends PendingEntry {public final BytesRef prefix;public final long fp;@@ -472,10 +491,8 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {assert scratchBytes.size() == 0;- // TODO: try writing the leading vLong in MSB order- // (opposite of what Lucene does today), for better- // outputs sharing in the FST- scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));+ // write the leading vLong in MSB order for better outputs sharing in the FST+ writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);if (isFloor) {scratchBytes.writeVInt(blocks.size() - 1);for (int i = 1; i < blocks.size(); i++) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.javaindex 0e865061cf0..cb5577d8d6c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java@@ -236,7 +236,7 @@ final class SegmentTermsEnum extends BaseTermsEnum {SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, BytesRef frameData, int length)throws IOException {scratchReader.reset(frameData.bytes, frameData.offset, frameData.length);- final long code = scratchReader.readVLong();+ final long code = fr.readVLongOutput(scratchReader);final long fpSeek = code >>> Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);f.hasTerms = (code & Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS) != 0;@@ -980,7 +980,7 @@ final class SegmentTermsEnum extends BaseTermsEnum {} else if (isSeekFrame && !f.isFloor) {final ByteArrayDataInput reader =new ByteArrayDataInput(output.bytes, output.offset, output.length);- final long codeOrig = reader.readVLong();+ final long codeOrig = fr.readVLongOutput(reader);final long code =(f.fp << Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)| (f.hasTerms ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/blocktree/TestMSBVLong.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/blocktree/TestMSBVLong.javanew file mode 100644index 00000000000..1ebab926209--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/blocktree/TestMSBVLong.java@@ -0,0 +1,44 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene90.blocktree;++import java.io.IOException;+import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.ByteArrayDataOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.ArrayUtil;++public class TestMSBVLong extends LuceneTestCase {++ public void testMSBVLong() throws IOException {+ assertMSBVLong(Long.MAX_VALUE);+ int iter = atLeast(10000);+ for (long i = 0; i < iter; i++) {+ assertMSBVLong(i);+ }+ }++ private static void assertMSBVLong(long l) throws IOException {+ byte[] bytes = new byte[10];+ ByteArrayDataOutput output = new ByteArrayDataOutput(bytes);+ Lucene90BlockTreeTermsWriter.writeMSBVLong(l, output);+ ByteArrayDataInput in =+ new ByteArrayDataInput(ArrayUtil.copyOfSubArray(bytes, 0, output.getPosition()));+ long recovered = FieldReader.readMSBVLong(in);+ assertEquals(l + " != " + recovered, l, recovered);+ }+}