Apache Lucene open-source search software
Bump block size of postings to 256. (#15160)
Adrien Grand00f2b01e7985
diff --git a/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle b/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradleindex 8facd4e438d..ec90f270bba 100644--- a/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle+++ b/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle@@ -47,9 +47,7 @@ void addGenerateFromPythonTask(Project project, File genDir, String taskName, Stconfigure(project(":lucene:core")) {addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/codecs/lucene103"), "ForUtil", "gen_ForUtil.py", "ForUtil.java")- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/codecs/lucene103"), "ForDeltaUtil", "gen_ForDeltaUtil.py", "ForDeltaUtil.java")+ file("src/java/org/apache/lucene/codecs/lucene104"), "ForUtil", "gen_ForUtil.py", "ForUtil.java")}configure(project(":lucene:backward-codecs")) {@@ -71,4 +69,9 @@ configure(project(":lucene:backward-codecs")) {file("src/java/org/apache/lucene/backward_codecs/lucene101"), "ForUtil101", "gen_ForUtil.py", "ForUtil.java")addGenerateFromPythonTask(project,file("src/java/org/apache/lucene/backward_codecs/lucene101"), "ForDeltaUtil101", "gen_ForDeltaUtil.py", "ForDeltaUtil.java")++ addGenerateFromPythonTask(project,+ file("src/java/org/apache/lucene/backward_codecs/lucene103"), "ForUtil103", "gen_ForUtil.py", "ForUtil.java")+ addGenerateFromPythonTask(project,+ file("src/java/org/apache/lucene/backward_codecs/lucene103"), "ForDeltaUtil103", "gen_ForDeltaUtil.py", "ForDeltaUtil.java")}diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 0ce8274e2b2..1c1b6450494 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -136,6 +136,9 @@ Optimizations* GITHUB#15151: Use `SimScorer#score` bulk API to compute impact scores perblock of postings. (Adrien Grand)+* GITHUB#15160: Increased the size used for blocks of postings from 128 to 256.+ This gives a noticeable speedup to many queries. (Adrien Grand)+Bug Fixes---------------------* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentExceptiondiff --git a/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil103.json b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil103.jsonnew file mode 100644index 00000000000..abe1d457c93--- /dev/null+++ b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil103.json@@ -0,0 +1,4 @@+{+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.java": "4cd0564cb34801443d050065ab7ae8d93e2dca92",+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.py": "8d656f27945b5a679babc41ac47afbd351359aa8"+}diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil103.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil103.jsonnew file mode 100644index 00000000000..cc5f05caa51--- /dev/null+++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil103.json@@ -0,0 +1,4 @@+{+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.java": "10904a43be9537b6c52ec13a1139c71c11345c97",+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.py": "61a6c7d73b0e8b166ebf7344fc24e95873240e9f"+}diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.javaindex 88cbd590951..5c8cf801669 100644--- a/lucene/backward-codecs/src/java/module-info.java+++ b/lucene/backward-codecs/src/java/module-info.java@@ -40,6 +40,7 @@ module org.apache.lucene.backward_codecs {exports org.apache.lucene.backward_codecs.lucene912;exports org.apache.lucene.backward_codecs.lucene100;exports org.apache.lucene.backward_codecs.lucene101;+ exports org.apache.lucene.backward_codecs.lucene103;exports org.apache.lucene.backward_codecs.packed;exports org.apache.lucene.backward_codecs.store;@@ -51,7 +52,8 @@ module org.apache.lucene.backward_codecs {org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,- org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat;+ org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat,+ org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat;provides org.apache.lucene.codecs.KnnVectorsFormat withorg.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,@@ -71,5 +73,6 @@ module org.apache.lucene.backward_codecs {org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,org.apache.lucene.backward_codecs.lucene100.Lucene100Codec,- org.apache.lucene.backward_codecs.lucene101.Lucene101Codec;+ org.apache.lucene.backward_codecs.lucene101.Lucene101Codec,+ org.apache.lucene.backward_codecs.lucene103.Lucene103Codec;}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/BitSetUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/BitSetUtil.javanew file mode 100644index 00000000000..fcbeca1ae69--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/BitSetUtil.java@@ -0,0 +1,104 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import java.util.Objects;+import org.apache.lucene.util.FixedBitSet;++class BitSetUtil {++ /**+ * Converts set bits in the given bitset to an array of document IDs. Only processes bits from+ * index {@code from} (inclusive) to {@code to} (exclusive) and returns the number of bits set in+ * this range.+ *+ * <p>Each set bit's position is converted to a document ID by adding the {@code base} value and+ * stored in the provided {@code array}.+ *+ * <p>NOTE: Caller need to ensure the {@code array} has a length greater than or equal to {@code+ * bitSet.cardinality(from, to) + 1}.+ */+ static int denseBitsetToArray(FixedBitSet bitSet, int from, int to, int base, int[] array) {+ assert bitSet.cardinality(from, to) + 1 <= array.length+ : "Array length must be at least bitSet.cardinality(from, to) + 1";++ Objects.checkFromToIndex(from, to, bitSet.length());++ int offset = 0;+ long[] bits = bitSet.getBits();+ // First, align `from` with a word start, ie. a multiple of Long.SIZE (64)+ if ((from & 0x3F) != 0) {+ long word = bits[from >> 6] >>> from;+ int numBitsTilNextWord = -from & 0x3F;+ if (to - from < numBitsTilNextWord) {+ // All bits are in a single word+ word &= (1L << (to - from)) - 1L;+ return word2Array(word, from + base, array, offset);+ }+ offset = word2Array(word, from + base, array, offset);+ from += numBitsTilNextWord;+ assert (from & 0x3F) == 0;+ }++ for (int i = from >> 6, end = to >> 6; i < end; ++i) {+ long word = bits[i];+ offset = word2Array(word, base + (i << 6), array, offset);+ }++ // Now handle remaining bits in the last partial word+ if ((to & 0x3F) != 0) {+ long word = bits[to >> 6] & ((1L << to) - 1);+ offset = word2Array(word, base + (to & ~0x3F), array, offset);+ }++ return offset;+ }++ private static int word2Array(long word, int base, int[] docs, int offset) {+ final int bitCount = Long.bitCount(word);++ if (bitCount >= 32) {+ return denseWord2Array(word, base, docs, offset);+ }++ for (int i = 0; i < bitCount; i++) {+ int ntz = Long.numberOfTrailingZeros(word);+ docs[offset++] = base + ntz;+ word ^= 1L << ntz;+ }++ return offset;+ }++ private static int denseWord2Array(long word, int base, int[] docs, int offset) {+ final int lWord = (int) word;+ final int hWord = (int) (word >>> 32);+ final int offset32 = offset + Integer.bitCount(lWord);+ int hOffset = offset32;++ for (int i = 0; i < 32; i++) {+ docs[offset] = base + i;+ docs[hOffset] = base + i + 32;+ offset += (lWord >>> i) & 1;+ hOffset += (hWord >>> i) & 1;+ }++ docs[offset32] = base + 32 + Integer.numberOfTrailingZeros(hWord);++ return hOffset;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.javasimilarity index 82%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.javaindex e577ce81aaf..481f04de566 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.java@@ -16,46 +16,45 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;-import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_1;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_2;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_4;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_5;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_6;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_7;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_8;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_1;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_10;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_11;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_12;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_13;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_14;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_15;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_16;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_2;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_3;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_4;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_5;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_6;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_7;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_8;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_9;-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse16;-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse8;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode1;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode10;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode2;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode3;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode9;-import static org.apache.lucene.codecs.lucene103.ForUtil.decodeSlow;-import static org.apache.lucene.codecs.lucene103.ForUtil.encode;-import static org.apache.lucene.codecs.lucene103.ForUtil.expand16;-import static org.apache.lucene.codecs.lucene103.ForUtil.expand8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_1;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_2;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_4;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_5;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_6;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_7;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_1;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_10;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_11;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_12;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_13;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_14;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_15;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_16;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_2;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_3;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_4;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_5;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_6;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_7;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_9;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse16;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode1;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode10;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode2;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode3;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode9;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decodeSlow;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.encode;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand16;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand8;import java.io.IOException;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;import org.apache.lucene.store.DataOutput;import org.apache.lucene.util.packed.PackedInts;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.javasimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.javaindex 4ccfbd28125..4e3a8a83b57 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.java@@ -16,10 +16,9 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.io.IOException;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;import org.apache.lucene.store.DataOutput;/**@@ -29,7 +28,9 @@ import org.apache.lucene.store.DataOutput;*/public final class ForUtil {+ /** Number of integers per block. */public static final int BLOCK_SIZE = 128;+static final int BLOCK_SIZE_LOG2 = 7;static int expandMask16(int mask16) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103Codec.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103Codec.javaindex d2e3eea6ef9..b571c12a5df 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103Codec.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.util.Objects;import org.apache.lucene.codecs.Codec;@@ -49,7 +49,7 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;** <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.*- * @see org.apache.lucene.codecs.lucene103 package documentation for file format details.+ * @see org.apache.lucene.backward_codecs.lucene103 package documentation for file format details.* @lucene.experimental*/public class Lucene103Codec extends Codec {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsFormat.javasimilarity index 94%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsFormat.javaindex 04880504311..9e9c8d3e7fc 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsFormat.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.io.IOException;import org.apache.lucene.codecs.BlockTermState;@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;-import org.apache.lucene.index.ImpactsEnum;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -319,7 +318,7 @@ import org.apache.lucene.util.packed.PackedInts;** @lucene.experimental*/-public final class Lucene103PostingsFormat extends PostingsFormat {+public class Lucene103PostingsFormat extends PostingsFormat {/** Filename extension for some small metadata about how postings are encoded. */public static final String META_EXTENSION = "psm";@@ -342,7 +341,7 @@ public final class Lucene103PostingsFormat extends PostingsFormat {/** Size of blocks. */public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;- public static final int BLOCK_MASK = BLOCK_SIZE - 1;+ static final int BLOCK_MASK = BLOCK_SIZE - 1;/** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */public static final int LEVEL1_FACTOR = 32;@@ -350,17 +349,7 @@ public final class Lucene103PostingsFormat extends PostingsFormat {/** Total number of docs covered by level 1 skip data: 32 * 128 = 4,096 */public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;- public static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;-- /**- * Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is- * internally used to help the JVM make good inlining decisions.- *- * @lucene.internal- */- public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {- return Lucene103PostingsReader.BlockPostingsEnum.class;- }+ static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;static final String TERMS_CODEC = "Lucene103PostingsWriterTerms";static final String META_CODEC = "Lucene103PostingsWriterMeta";@@ -372,10 +361,6 @@ public final class Lucene103PostingsFormat extends PostingsFormat {static final int VERSION_CURRENT = VERSION_START;- private final int version;- private final int minTermBlockSize;- private final int maxTermBlockSize;-/** Creates {@code Lucene103PostingsFormat} with default settings. */public Lucene103PostingsFormat() {this(@@ -391,31 +376,13 @@ public final class Lucene103PostingsFormat extends PostingsFormat {* Lucene103BlockTreeTermsWriter#Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)*/public Lucene103PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {- this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);- }-- /** Expert constructor that allows setting the version. */- public Lucene103PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {super("Lucene103");- if (version < VERSION_START || version > VERSION_CURRENT) {- throw new IllegalArgumentException("Version out of range: " + version);- }- this.version = version;- Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);- this.minTermBlockSize = minTermBlockSize;- this.maxTermBlockSize = maxTermBlockSize;}@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state, version);- try {- return new Lucene103BlockTreeTermsWriter(- state, postingsWriter, minTermBlockSize, maxTermBlockSize);- } catch (Throwable t) {- IOUtils.closeWhileSuppressingExceptions(t, postingsWriter);- throw t;- }+ throw new UnsupportedOperationException(+ "This postings format may not be used for writing, use the current postings format");}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.javasimilarity index 96%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.javaindex 1d5c71167ed..84d50d551cd 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.java@@ -14,24 +14,24 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;--import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.LEVEL1_NUM_DOCS;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.META_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.VERSION_CURRENT;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.VERSION_START;+package org.apache.lucene.backward_codecs.lucene103;++import static org.apache.lucene.backward_codecs.lucene103.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.LEVEL1_NUM_DOCS;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.META_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.VERSION_CURRENT;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.VERSION_START;import java.io.IOException;import java.util.Arrays;+import org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;import org.apache.lucene.codecs.BlockTermState;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FreqAndNormBuffer;import org.apache.lucene.index.Impacts;@@ -40,8 +40,6 @@ import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;-import org.apache.lucene.internal.vectorization.VectorizationProvider;import org.apache.lucene.search.DocAndFloatFeatureBuffer;import org.apache.lucene.store.ByteArrayDataInput;import org.apache.lucene.store.ChecksumIndexInput;@@ -63,8 +61,6 @@ import org.apache.lucene.util.VectorUtil;*/public final class Lucene103PostingsReader extends PostingsReaderBase {- static final VectorizationProvider VECTORIZATION_PROVIDER = VectorizationProvider.getInstance();-private final IndexInput docIn;private final IndexInput posIn;private final IndexInput payIn;@@ -452,7 +448,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {if (needsPos) {this.posIn = Lucene103PostingsReader.this.posIn.clone();- posInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(posIn);+ posInUtil = new PostingDecodingUtil(posIn);posDeltaBuffer = new int[BLOCK_SIZE];} else {this.posIn = null;@@ -462,7 +458,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {if (needsOffsets || needsPayloads) {this.payIn = Lucene103PostingsReader.this.payIn.clone();- payInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(payIn);+ payInUtil = new PostingDecodingUtil(payIn);} else {this.payIn = null;payInUtil = null;@@ -505,7 +501,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {if (docIn == null) {// lazy initdocIn = Lucene103PostingsReader.this.docIn.clone();- docInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(docIn);+ docInUtil = new PostingDecodingUtil(docIn);}prefetchPostings(docIn, termState);}@@ -1395,9 +1391,6 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {}}- /**- * @see Lucene103PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)- */static int readVInt15(DataInput in) throws IOException {short s = in.readShort();if (s >= 0) {@@ -1407,9 +1400,6 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {}}- /**- * @see Lucene103PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)- */static long readVLong15(DataInput in) throws IOException {short s = in.readShort();if (s >= 0) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PForUtil.javasimilarity index 97%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PForUtil.javaindex 6c8600a6e81..e7d8e0300ee 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PForUtil.java@@ -14,11 +14,10 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.io.IOException;import java.util.Arrays;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;import org.apache.lucene.store.DataInput;import org.apache.lucene.store.DataOutput;import org.apache.lucene.util.packed.PackedInts;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingDecodingUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingDecodingUtil.javanew file mode 100644index 00000000000..99f3c01f44d--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingDecodingUtil.java@@ -0,0 +1,65 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import java.io.IOException;+import org.apache.lucene.store.IndexInput;++/** Utility class to decode postings. */+final class PostingDecodingUtil {++ /** The wrapper {@link IndexInput}. */+ public final IndexInput in;++ /** Sole constructor, called by sub-classes. */+ PostingDecodingUtil(IndexInput in) {+ this.in = in;+ }++ /**+ * Core methods for decoding blocks of docs / freqs / positions / offsets.+ *+ * <ul>+ * <li>Read {@code count} ints.+ * <li>For all {@code i} >= 0 so that {@code bShift - i * dec} > 0, apply shift {@code+ * bShift - i * dec} and store the result in {@code b} at offset {@code count * i}.+ * <li>Apply mask {@code cMask} and store the result in {@code c} starting at offset {@code+ * cIndex}.+ * </ul>+ */+ public void splitInts(+ int count, int[] b, int bShift, int dec, int bMask, int[] c, int cIndex, int cMask)+ throws IOException {+ in.readInts(c, cIndex, count);+ final int maxIter = (bShift - 1) / dec;++ // Process each shift level across all elements (better for vectorization)+ for (int j = 0; j <= maxIter; ++j) {+ final int shift = bShift - j * dec;+ final int bOffset = count * j;+ // Vectorizable loop: contiguous memory access with simple operations+ for (int i = 0; i < count; ++i) {+ b[bOffset + i] = (c[cIndex + i] >>> shift) & bMask;+ }+ }++ // Apply mask to c array (vectorizable)+ for (int i = 0; i < count; ++i) {+ c[cIndex + i] &= cMask;+ }+ }+}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingsUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingsUtil.javanew file mode 100644index 00000000000..1038e32a93c--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingsUtil.java@@ -0,0 +1,74 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import java.io.IOException;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.GroupVIntUtil;++/** Utility class to encode/decode postings block. */+final class PostingsUtil {++ /**+ * Read values that have been written using variable-length encoding and group-varint encoding+ * instead of bit-packing.+ */+ static void readVIntBlock(+ IndexInput docIn,+ int[] docBuffer,+ int[] freqBuffer,+ int num,+ boolean indexHasFreq,+ boolean decodeFreq)+ throws IOException {+ GroupVIntUtil.readGroupVInts(docIn, docBuffer, num);+ if (indexHasFreq && decodeFreq) {+ for (int i = 0; i < num; ++i) {+ freqBuffer[i] = docBuffer[i] & 0x01;+ docBuffer[i] >>>= 1;+ if (freqBuffer[i] == 0) {+ freqBuffer[i] = docIn.readVInt();+ }+ }+ } else if (indexHasFreq) {+ for (int i = 0; i < num; ++i) {+ docBuffer[i] >>>= 1;+ }+ }+ }++ /** Write freq buffer with variable-length encoding and doc buffer with group-varint encoding. */+ static void writeVIntBlock(+ DataOutput docOut, int[] docBuffer, int[] freqBuffer, int num, boolean writeFreqs)+ throws IOException {+ if (writeFreqs) {+ for (int i = 0; i < num; i++) {+ docBuffer[i] = (docBuffer[i] << 1) | (freqBuffer[i] == 1 ? 1 : 0);+ }+ }+ docOut.writeGroupVInts(docBuffer, num);+ if (writeFreqs) {+ for (int i = 0; i < num; i++) {+ final int freq = freqBuffer[i];+ if (freq != 1) {+ docOut.writeVInt(freq);+ }+ }+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.pysimilarity index 78%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.pyrename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.pyindex 290a6a29e23..912f18e6ce7 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.py@@ -40,48 +40,47 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.io.IOException;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;import org.apache.lucene.store.DataOutput;import org.apache.lucene.util.packed.PackedInts;-import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_1;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_2;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_4;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_5;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_6;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_7;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_8;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_1;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_10;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_11;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_12;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_13;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_14;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_15;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_16;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_2;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_3;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_4;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_5;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_6;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_7;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_8;-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_9;-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse16;-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse8;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode1;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode10;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode2;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode3;-import static org.apache.lucene.codecs.lucene103.ForUtil.decode9;-import static org.apache.lucene.codecs.lucene103.ForUtil.decodeSlow;-import static org.apache.lucene.codecs.lucene103.ForUtil.encode;-import static org.apache.lucene.codecs.lucene103.ForUtil.expand16;-import static org.apache.lucene.codecs.lucene103.ForUtil.expand8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_1;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_2;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_4;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_5;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_6;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_7;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_1;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_10;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_11;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_12;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_13;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_14;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_15;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_16;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_2;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_3;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_4;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_5;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_6;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_7;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_9;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse16;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse8;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode1;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode10;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode2;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode3;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode9;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decodeSlow;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.encode;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand16;+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand8;/*** Inspired from https://fulmicoton.com/posts/bitpacking/diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.pysimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.pyrename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.pyindex f3bb9fb50cd..6ecfeee16d0 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.py@@ -40,10 +40,9 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.io.IOException;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;import org.apache.lucene.store.DataOutput;/**@@ -55,6 +54,7 @@ import org.apache.lucene.store.DataOutput;*/public final class ForUtil {+ /** Number of integers per block. */public static final int BLOCK_SIZE = 128;static final int BLOCK_SIZE_LOG2 = 7;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/package-info.javanew file mode 100644index 00000000000..f23a48fcccf--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/package-info.java@@ -0,0 +1,436 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/**+ * Lucene 10.3 file format.+ *+ * <h2>Apache Lucene - Index File Formats</h2>+ *+ * <div>+ *+ * <ul>+ * <li><a href="#Introduction">Introduction</a>+ * <li><a href="#Definitions">Definitions</a>+ * <ul>+ * <li><a href="#Inverted_Indexing">Inverted Indexing</a>+ * <li><a href="#Types_of_Fields">Types of Fields</a>+ * <li><a href="#Segments">Segments</a>+ * <li><a href="#Document_Numbers">Document Numbers</a>+ * </ul>+ * <li><a href="#Overview">Index Structure Overview</a>+ * <li><a href="#File_Naming">File Naming</a>+ * <li><a href="#file-names">Summary of File Extensions</a>+ * <ul>+ * <li><a href="#Lock_File">Lock File</a>+ * <li><a href="#History">History</a>+ * <li><a href="#Limitations">Limitations</a>+ * </ul>+ * </ul>+ *+ * </div> <a id="Introduction"></a>+ *+ * <h3>Introduction</h3>+ *+ * <div>+ *+ * <p>This document defines the index file formats used in this version of Lucene. If you are using+ * a different version of Lucene, please consult the copy of <code>docs/</code> that was distributed+ * with the version you are using.+ *+ * <p>This document attempts to provide a high-level definition of the Apache Lucene file formats.+ * </div> <a id="Definitions"></a>+ *+ * <h3>Definitions</h3>+ *+ * <div>+ *+ * <p>The fundamental concepts in Lucene are index, document, field and term.+ *+ * <p>An index contains a sequence of documents.+ *+ * <ul>+ * <li>A document is a sequence of fields.+ * <li>A field is a named sequence of terms.+ * <li>A term is a sequence of bytes.+ * </ul>+ *+ * <p>The same sequence of bytes in two different fields is considered a different term. Thus terms+ * are represented as a pair: the string naming the field, and the bytes within the field. <a+ * id="Inverted_Indexing"></a>+ *+ * <h4>Inverted Indexing</h4>+ *+ * <p>Lucene's index stores terms and statistics about those terms in order to make term-based+ * search more efficient. Lucene's terms index falls into the family of indexes known as an+ * <i>inverted index.</i> This is because it can list, for a term, the documents that contain it.+ * This is the inverse of the natural relationship, in which documents list terms. <a+ * id="Types_of_Fields"></a>+ *+ * <h4>Types of Fields</h4>+ *+ * <p>In Lucene, fields may be <i>stored</i>, in which case their text is stored in the index+ * literally, in a non-inverted manner. Fields that are inverted are called <i>indexed</i>. A field+ * may be both stored and indexed.+ *+ * <p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the text of a field+ * may be used literally as a term to be indexed. Most fields are tokenized, but sometimes it is+ * useful for certain identifier fields to be indexed literally.+ *+ * <p>See the {@link org.apache.lucene.document.Field Field} java docs for more information on+ * Fields. <a id="Segments"></a>+ *+ * <h4>Segments</h4>+ *+ * <p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>. Each segment is a+ * fully independent index, which could be searched separately. Indexes evolve by:+ *+ * <ol>+ * <li>Creating new segments for newly added documents.+ * <li>Merging existing segments.+ * </ol>+ *+ * <p>Searches may involve multiple segments and/or multiple indexes, each index potentially+ * composed of a set of segments. <a id="Document_Numbers"></a>+ *+ * <h4>Document Numbers</h4>+ *+ * <p>Internally, Lucene refers to documents by an integer <i>document number</i>. The first+ * document added to an index is numbered zero, and each subsequent document added gets a number one+ * greater than the previous.+ *+ * <p>Note that a document's number may change, so caution should be taken when storing these+ * numbers outside of Lucene. In particular, numbers may change in the following situations:+ *+ * <ul>+ * <li>+ * <p>The numbers stored in each segment are unique only within the segment, and must be+ * converted before they can be used in a larger context. The standard technique is to+ * allocate each segment a range of values, based on the range of numbers used in that+ * segment. To convert a document number from a segment to an external value, the segment's+ * <i>base</i> document number is added. To convert an external value back to a+ * segment-specific value, the segment is identified by the range that the external value is+ * in, and the segment's base value is subtracted. For example two five document segments+ * might be combined, so that the first segment has a base value of zero, and the second of+ * five. Document three from the second segment would have an external value of eight.+ * <li>+ * <p>When documents are deleted, gaps are created in the numbering. These are eventually+ * removed as the index evolves through merging. Deleted documents are dropped when segments+ * are merged. A freshly-merged segment thus has no gaps in its numbering.+ * </ul>+ *+ * </div> <a id="Overview"></a>+ *+ * <h3>Index Structure Overview</h3>+ *+ * <div>+ *+ * <p>Each segment index maintains the following:+ *+ * <ul>+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment info}. This+ * contains metadata about a segment, such as the number of documents, what files it uses, and+ * information about how the segment is sorted+ * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Field names}. This+ * contains metadata about the set of named fields used in the index.+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Stored Field values}.+ * This contains, for each document, a list of attribute-value pairs, where the attributes are+ * field names. These are used to store auxiliary information about the document, such as its+ * title, url, or an identifier to access a database. The set of stored fields are what is+ * returned for each hit when searching. This is keyed by document number.+ * <li>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term+ * dictionary}. A dictionary containing all of the terms used in all of the indexed fields of+ * all of the documents. The dictionary also contains the number of documents which contain+ * the term, and pointers to the term's frequency and proximity data.+ * <li>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Frequency+ * data}. For each term in the dictionary, the numbers of all the documents that contain that+ * term, and the frequency of the term in that document, unless frequencies are omitted+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})+ * <li>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Proximity+ * data}. For each term in the dictionary, the positions that the term occurs in each+ * document. Note that this will not exist if all fields in all documents omit position data.+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For+ * each field in each document, a value is stored that is multiplied into the score for hits+ * on that field.+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vectors}. For each+ * field in each document, the term vector (sometimes called document vector) may be stored. A+ * term vector consists of term text and term frequency. To add Term Vectors to your index see+ * the {@link org.apache.lucene.document.Field Field} constructors+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-document values}. Like+ * stored values, these are also keyed by document number, but are generally intended to be+ * loaded into main memory for fast access. Whereas stored values are generally intended for+ * summary results from searches, per-document values are useful for things like scoring+ * factors.+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live documents}. An+ * optional file indicating which documents are live.+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}. Optional pair+ * of files, recording dimensionally indexed fields, to enable fast numeric range filtering+ * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape+ * intersection (2D, 3D).+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}. The+ * vector format stores numeric vectors in a format optimized for random access and+ * computation, supporting high-dimensional nearest-neighbor search.+ * </ul>+ *+ * <p>Details on each of these are provided in their linked pages. </div> <a id="File_Naming"></a>+ *+ * <h3>File Naming</h3>+ *+ * <div>+ *+ * <p>All files belonging to a segment have the same name with varying extensions. The extensions+ * correspond to the different file formats described below. When using the Compound File format+ * (default for small segments) these files (except for the Segment info file, the Lock file, and+ * Deleted documents file) are collapsed into a single .cfs file (see below for details)+ *+ * <p>Typically, all segments in an index are stored in a single directory, although this is not+ * required.+ *+ * <p>File names are never re-used. That is, when any file is saved to the Directory it is given a+ * never before used filename. This is achieved using a simple generations approach. For example,+ * the first segments file is segments_1, then segments_2, etc. The generation is a sequential long+ * integer represented in alpha-numeric (base 36) form. </div> <a id="file-names"></a>+ *+ * <h3>Summary of File Extensions</h3>+ *+ * <div>+ *+ * <p>The following table summarizes the names and extensions of the files in Lucene:+ *+ * <table class="padding4" style="border-spacing: 1px; border-collapse: separate">+ * <caption>lucene filenames by extension</caption>+ * <tr>+ * <th>Name</th>+ * <th>Extension</th>+ * <th>Brief Description</th>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>+ * <td>segments_N</td>+ * <td>Stores information about a commit point</td>+ * </tr>+ * <tr>+ * <td><a href="#Lock_File">Lock File</a></td>+ * <td>write.lock</td>+ * <td>The Write lock prevents multiple IndexWriters from writing to the same+ * file.</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment Info}</td>+ * <td>.si</td>+ * <td>Stores metadata about a segment</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat Compound File}</td>+ * <td>.cfs, .cfe</td>+ * <td>An optional "virtual" file consisting of all the other index files for+ * systems that frequently run out of file handles.</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Fields}</td>+ * <td>.fnm</td>+ * <td>Stores information about the fields</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Index}</td>+ * <td>.fdx</td>+ * <td>Contains pointers to field data</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Data}</td>+ * <td>.fdt</td>+ * <td>The stored fields for documents</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>+ * <td>.tim</td>+ * <td>The term dictionary, stores term info</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Index}</td>+ * <td>.tip</td>+ * <td>The index into the Term Dictionary</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>+ * <td>.doc</td>+ * <td>Contains the list of docs which contain each term along with frequency</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Positions}</td>+ * <td>.pos</td>+ * <td>Stores position information about where a term occurs in the index</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Payloads}</td>+ * <td>.pay</td>+ * <td>Stores additional per-position metadata information such as character offsets and user payloads</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Norms}</td>+ * <td>.nvd, .nvm</td>+ * <td>Encodes length and boost factors for docs and fields</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-Document Values}</td>+ * <td>.dvd, .dvm</td>+ * <td>Encodes additional scoring factors or other per-document information.</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Index}</td>+ * <td>.tvx</td>+ * <td>Stores offset into the document data file</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Data}</td>+ * <td>.tvd</td>+ * <td>Contains term vector data.</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live Documents}</td>+ * <td>.liv</td>+ * <td>Info about what documents are live</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}</td>+ * <td>.kdd, .kdi, .kdm</td>+ * <td>Holds indexed points</td>+ * </tr>+ * <tr>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}</td>+ * <td>.vec, .vem, .veq, vex</td>+ * <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data,+ * <code>.vem</code> the vector metadata, <code>.veq</code> the quantized vector data, and <code>.vex</code> the+ * hnsw graph data.</td>+ * </tr>+ * </table>+ *+ * </div> <a id="Lock_File"></a>+ *+ * <h3>Lock File</h3>+ *+ * The write lock, which is stored in the index directory by default, is named "write.lock". If the+ * lock directory is different from the index directory then the write lock will be named+ * "XXXX-write.lock" where XXXX is a unique prefix derived from the full path to the index+ * directory. When this file is present, a writer is currently modifying the index (adding or+ * removing documents). This lock file ensures that only one writer is modifying the index at a+ * time. <a id="History"></a>+ *+ * <h3>History</h3>+ *+ * <p>Compatibility notes are provided in this document, describing how file formats have changed+ * from prior versions:+ *+ * <ul>+ * <li>In version 2.1, the file format was changed to allow lock-less commits (ie, no more commit+ * lock). The change is fully backwards compatible: you can open a pre-2.1 index for searching+ * or adding/deleting of docs. When the new segments file is saved (committed), it will be+ * written in the new file format (meaning no specific "upgrade" process is needed). But note+ * that once a commit has occurred, pre-2.1 Lucene will not be able to read the index.+ * <li>In version 2.3, the file format was changed to allow segments to share a single set of doc+ * store (vectors & stored fields) files. This allows for faster indexing in certain+ * cases. The change is fully backwards compatible (in the same way as the lock-less commits+ * change in 2.1).+ * <li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not Java's modified+ * UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">LUCENE-510</a> for+ * details.+ * <li>In version 2.9, an optional opaque Map<String,String> CommitUserData may be passed to+ * IndexWriter's commit methods (and later retrieved), which is recorded in the segments_N+ * file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">LUCENE-1382</a> for+ * details. Also, diagnostics were added to each segment written recording details about why+ * it was written (due to flush, merge; which OS/JRE was used; etc.). See issue <a+ * href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.+ * <li>In version 3.0, compressed fields are no longer written to the index (they can still be+ * read, but on merge the new segment will write them, uncompressed). See issue <a+ * href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a> for details.+ * <li>In version 3.1, segments records the code version that created them. See <a+ * href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.+ * Additionally segments track explicitly whether or not they have term vectors. See <a+ * href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a> for details.+ * <li>In version 3.2, numeric fields are written as natively to stored fields file, previously+ * they were stored in text format only.+ * <li>In version 3.4, fields can omit position data while still indexing term frequencies.+ * <li>In version 4.0, the format of the inverted index became extensible via the {@link+ * org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage ({@code DocValues})+ * was introduced. Normalization factors need no longer be a single byte, they can be any+ * {@link org.apache.lucene.index.NumericDocValues NumericDocValues}. Terms need not be+ * unicode strings, they can be any byte sequence. Term offsets can optionally be indexed into+ * the postings lists. Payloads can be stored in the term vectors.+ * <li>In version 4.1, the format of the postings list changed to use either of FOR compression or+ * variable-byte encoding, depending upon the frequency of the term. Terms appearing only once+ * were changed to inline directly into the term dictionary. Stored fields are compressed by+ * default.+ * <li>In version 4.2, term vectors are compressed by default. DocValues has a new multi-valued+ * type (SortedSet), that can be used for faceting/grouping/joining on multi-valued fields.+ * <li>In version 4.5, DocValues were extended to explicitly represent missing values.+ * <li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to+ * allow updating NumericDocValues fields.+ * <li>In version 4.8, checksum footers were added to the end of each index file for improved data+ * integrity. Specifically, the last 8 bytes of every index file contain the zlib-crc32+ * checksum of the file.+ * <li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric) that is+ * suitable for faceting/sorting/analytics.+ * <li>In version 5.4, DocValues have been improved to store more information on disk: addresses+ * for binary fields and ord indexes for multi-valued fields.+ * <li>In version 6.0, Points were added, for multi-dimensional range/distance search.+ * <li>In version 6.2, new Segment info format that reads/writes the index sort, to support index+ * sorting.+ * <li>In version 7.0, DocValues have been improved to better support sparse doc values thanks to+ * an iterator API.+ * <li>In version 8.0, postings have been enhanced to record, for each block of doc ids, the (term+ * freq, normalization factor) pairs that may trigger the maximum score of the block. This+ * information is recorded alongside skip data in order to be able to skip blocks of doc ids+ * if they may not produce high enough scores. Additionally doc values and norms has been+ * extended with jump-tables to make access O(1) instead of O(n), where n is the number of+ * elements to skip when advancing in the data.+ * <li>In version 8.4, postings, positions, offsets and payload lengths have move to a more+ * performant encoding that is vectorized.+ * <li>In version 8.6, index sort serialization is delegated to the sorts themselves, to allow+ * user-defined sorts to be used+ * <li>In version 8.6, points fields split the index tree and leaf data into separate files, to+ * allow for different access patterns to the different data structures+ * <li>In version 8.7, stored fields compression became adaptive to better handle documents with+ * smaller stored fields.+ * <li>In version 9.0, vector-valued fields were added.+ * <li>In version 9.1, vector-valued fields were modified to add a graph hierarchy.+ * <li>In version 9.2, docs of vector-valued fields were moved from .vem to .vec and encoded by+ * IndexDISI. ordToDoc mappings was added to .vem.+ * <li>In version 9.5, HNSW graph connections were changed to be delta-encoded with vints.+ * Additionally, metadata file size improvements were made by delta-encoding nodes by graph+ * layer and not writing the node ids for the zeroth layer.+ * <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector+ * format to utilize int8 quantized vectors for float32 vector search.+ * <li>In version 9.12, skip data was refactored to have only two levels: every 128 docs and every+ * 4,06 docs, and to be inlined in postings lists. This resulted in a speedup for queries that+ * need skipping, especially conjunctions.+ * <li>In version 10.1, block encoding changed to be optimized for int[] storage instead of+ * long[].+ * <li>In version 10.3, the index of block tree changed to be specialized trie instead of FST.+ * </ul>+ *+ * <a id="Limitations"></a>+ *+ * <h3>Limitations</h3>+ *+ * <div>+ *+ * <p>Lucene uses a Java <code>int</code> to refer to document numbers, and the index file format+ * uses an <code>Int32</code> on-disk to store document numbers. This is a limitation of both the+ * index file format and the current implementation. Eventually these should be replaced with either+ * <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt+ * VInt} values which have no limit. </div>+ */+package org.apache.lucene.backward_codecs.lucene103;diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codecindex 9f6cf615a3e..15bb6e9fa56 100644--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec@@ -26,3 +26,4 @@ org.apache.lucene.backward_codecs.lucene99.Lucene99Codecorg.apache.lucene.backward_codecs.lucene912.Lucene912Codecorg.apache.lucene.backward_codecs.lucene100.Lucene100Codecorg.apache.lucene.backward_codecs.lucene101.Lucene101Codec+org.apache.lucene.backward_codecs.lucene103.Lucene103Codecdiff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex 84cf4182384..a0c1266b755 100644--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -19,3 +19,4 @@ org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormatorg.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormatorg.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormatorg.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat+org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormatdiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsWriter.javasimilarity index 97%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsWriter.javaindex d49e6a0f2e1..7e733340473 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsWriter.java@@ -14,26 +14,26 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.LEVEL1_MASK;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.META_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.LEVEL1_MASK;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.META_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;import java.io.IOException;import java.util.Arrays;import java.util.Collection;import java.util.List;+import org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;import org.apache.lucene.codecs.BlockTermState;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.CompetitiveImpactAccumulator;import org.apache.lucene.codecs.Impact;import org.apache.lucene.codecs.PushPostingsWriterBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexFileNames;diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103RWPostingsFormat.javanew file mode 100644index 00000000000..1796325cdc1--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103RWPostingsFormat.java@@ -0,0 +1,75 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import java.io.IOException;+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;+import org.apache.lucene.codecs.FieldsConsumer;+import org.apache.lucene.codecs.PostingsWriterBase;+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.util.IOUtils;++/** Read-write impersonation of {@link Lucene103PostingsFormat}. */+public final class Lucene103RWPostingsFormat extends Lucene103PostingsFormat {++ private final int version;+ private final int minTermBlockSize;+ private final int maxTermBlockSize;++ /** Creates {@code Lucene103PostingsFormat} with default settings. */+ public Lucene103RWPostingsFormat() {+ this(+ Lucene103BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,+ Lucene103BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);+ }++ /**+ * Creates {@code Lucene103PostingsFormat} with custom values for {@code minBlockSize} and {@code+ * maxBlockSize} passed to block terms dictionary.+ *+ * @see+ * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)+ */+ public Lucene103RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {+ this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);+ }++ /** Expert constructor that allows setting the version. */+ public Lucene103RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {+ super();+ if (version < VERSION_START || version > VERSION_CURRENT) {+ throw new IllegalArgumentException("Version out of range: " + version);+ }+ this.version = version;+ Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);+ this.minTermBlockSize = minTermBlockSize;+ this.maxTermBlockSize = maxTermBlockSize;+ }++ @Override+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state, version);+ try {+ return new Lucene103BlockTreeTermsWriter(+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, postingsWriter);+ throw t;+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestBitSetUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestBitSetUtil.javanew file mode 100644index 00000000000..094d2111f3a--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestBitSetUtil.java@@ -0,0 +1,53 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;++import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.FixedBitSet;++public class TestBitSetUtil extends LuceneTestCase {++ public void testDenseBitsetToArray() throws Exception {+ for (int iter = 0; iter < 1000; iter++) {+ FixedBitSet bitSet = new FixedBitSet(BLOCK_SIZE + random().nextInt(200));+ while (bitSet.cardinality() < BLOCK_SIZE) {+ bitSet.set(random().nextInt(bitSet.length()));+ }++ int[] array = new int[BLOCK_SIZE + 1];+ int from, to, size;+ if (random().nextBoolean()) {+ from = 0;+ to = bitSet.length();+ } else {+ from = random().nextInt(bitSet.length());+ to = from + random().nextInt(bitSet.length() - from);+ }++ int base = random().nextInt(1000);+ size = BitSetUtil.denseBitsetToArray(bitSet, from, to, base, array);++ assertEquals(bitSet.cardinality(from, to), size);++ int[] index = new int[] {0};+ bitSet.forEach(from, to, base, bit -> assertEquals(bit, array[index[0]++]));+ assertEquals(size, index[0]);+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForDeltaUtil.javasimilarity index 93%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForDeltaUtil.javaindex 224731bdaf1..5fa53436adc 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForDeltaUtil.java@@ -14,12 +14,11 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import com.carrotsearch.randomizedtesting.generators.RandomNumbers;import java.io.IOException;import java.util.Arrays;-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;import org.apache.lucene.store.ByteBuffersDirectory;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;@@ -67,8 +66,7 @@ public class TestForDeltaUtil extends LuceneTestCase {{// decodeIndexInput in = d.openInput("test.bin", IOContext.READONCE);- PostingDecodingUtil pdu =- Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);ForDeltaUtil forDeltaUtil = new ForDeltaUtil();for (int i = 0; i < iterations; ++i) {int base = 0;diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForUtil.javanew file mode 100644index 00000000000..8cdada44475--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForUtil.java@@ -0,0 +1,95 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.packed.PackedInts;++public class TestForUtil extends LuceneTestCase {++ public void testEncodeDecode() throws IOException {+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];++ for (int i = 0; i < iterations; ++i) {+ final int bpv = TestUtil.nextInt(random(), 1, 31);+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ values[i * ForUtil.BLOCK_SIZE + j] =+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));+ }+ }++ final Directory d = new ByteBuffersDirectory();+ final long endPointer;++ {+ // encode+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);+ final ForUtil forUtil = new ForUtil();++ for (int i = 0; i < iterations; ++i) {+ int[] source = new int[ForUtil.BLOCK_SIZE];+ long or = 0;+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];+ or |= source[j];+ }+ final int bpv = PackedInts.bitsRequired(or);+ out.writeByte((byte) bpv);+ forUtil.encode(source, bpv, out);+ }+ endPointer = out.getFilePointer();+ out.close();+ }++ {+ // decode+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);+ ForUtil forUtil = new ForUtil();+ for (int i = 0; i < iterations; ++i) {+ final int bitsPerValue = in.readByte();+ final long currentFilePointer = in.getFilePointer();+ final int[] restored = new int[ForUtil.BLOCK_SIZE];+ forUtil.decode(bitsPerValue, pdu, restored);+ int[] ints = new int[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ ints[j] = Math.toIntExact(restored[j]);+ }+ assertArrayEquals(+ Arrays.toString(ints),+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),+ ints);+ assertEquals(ForUtil.numBytes(bitsPerValue), in.getFilePointer() - currentFilePointer);+ }+ assertEquals(endPointer, in.getFilePointer());+ in.close();+ }++ d.close();+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestLucene103PostingsFormat.javasimilarity index 98%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestLucene103PostingsFormat.javaindex 4bceeaae6ee..49538ce371e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestLucene103PostingsFormat.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.backward_codecs.lucene103;import java.io.IOException;import java.util.ArrayList;@@ -46,7 +46,7 @@ public class TestLucene103PostingsFormat extends BasePostingsFormatTestCase {@Overrideprotected Codec getCodec() {- return TestUtil.alwaysPostingsFormat(new Lucene103PostingsFormat());+ return TestUtil.alwaysPostingsFormat(new Lucene103RWPostingsFormat());}public void testVInt15() throws IOException {diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPForUtil.javanew file mode 100644index 00000000000..6d13212bf1a--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPForUtil.java@@ -0,0 +1,105 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.packed.PackedInts;++public class TestPForUtil extends LuceneTestCase {++ public void testEncodeDecode() throws IOException {+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);+ final int[] values = createTestData(iterations, 31);++ final Directory d = new ByteBuffersDirectory();+ final long endPointer = encodeTestData(iterations, values, d);++ IndexInput in = d.openInput("test.bin", IOContext.READONCE);+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);+ final PForUtil pforUtil = new PForUtil();+ for (int i = 0; i < iterations; ++i) {+ if (random().nextInt(5) == 0) {+ PForUtil.skip(in);+ continue;+ }+ final int[] restored = new int[ForUtil.BLOCK_SIZE];+ pforUtil.decode(pdu, restored);+ int[] ints = new int[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ ints[j] = Math.toIntExact(restored[j]);+ }+ assertArrayEquals(+ Arrays.toString(ints),+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),+ ints);+ }+ assertEquals(endPointer, in.getFilePointer());+ in.close();++ d.close();+ }++ private int[] createTestData(int iterations, int maxBpv) {+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];++ for (int i = 0; i < iterations; ++i) {+ final int bpv = TestUtil.nextInt(random(), 0, maxBpv);+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ values[i * ForUtil.BLOCK_SIZE + j] =+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));+ if (random().nextInt(100) == 0) {+ final int exceptionBpv;+ if (random().nextInt(10) == 0) {+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 9, 16), maxBpv);+ } else {+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 1, 8), maxBpv);+ }+ values[i * ForUtil.BLOCK_SIZE + j] |= random().nextInt(1 << (exceptionBpv - bpv)) << bpv;+ }+ }+ }++ return values;+ }++ private long encodeTestData(int iterations, int[] values, Directory d) throws IOException {+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);+ final PForUtil pforUtil = new PForUtil();++ for (int i = 0; i < iterations; ++i) {+ int[] source = new int[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];+ }+ pforUtil.encode(source, out);+ }+ final long endPointer = out.getFilePointer();+ out.close();++ return endPointer;+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPostingsUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPostingsUtil.javanew file mode 100644index 00000000000..cce53a9f87e--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPostingsUtil.java@@ -0,0 +1,57 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene103;++import java.io.IOException;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestPostingsUtil extends LuceneTestCase {++ // checks for bug described in https://github.com/apache/lucene/issues/13373+ public void testIntegerOverflow() throws IOException {+ // Size that writes the first value as a regular vint+ int randomSize1 = random().nextInt(1, 3);+ // Size that writes the first value as a group vint+ int randomSize2 = random().nextInt(4, ForUtil.BLOCK_SIZE);+ doTestIntegerOverflow(randomSize1);+ doTestIntegerOverflow(randomSize2);+ }++ private void doTestIntegerOverflow(int size) throws IOException {+ final int[] docDeltaBuffer = new int[size];+ final int[] freqBuffer = new int[size];++ final int delta = 1 << 30;+ docDeltaBuffer[0] = delta;+ try (Directory dir = newDirectory()) {+ try (IndexOutput out = dir.createOutput("test", IOContext.DEFAULT)) {+ // In old implementation, this would cause integer overflow exception.+ PostingsUtil.writeVIntBlock(out, docDeltaBuffer, freqBuffer, size, true);+ }+ int[] restoredDocs = new int[size];+ int[] restoredFreqs = new int[size];+ try (IndexInput in = dir.openInput("test", IOContext.DEFAULT)) {+ PostingsUtil.readVIntBlock(in, restoredDocs, restoredFreqs, size, true, true);+ }+ assertEquals(delta, restoredDocs[0]);+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.javaindex a2c09f53526..384ce889c5a 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java@@ -21,9 +21,7 @@ import static org.apache.lucene.backward_index.TestBasicBackwardsCompatibility.aimport com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import java.io.IOException;import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;-import org.apache.lucene.codecs.lucene103.Lucene103Codec;import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;@@ -69,14 +67,10 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTe}protected Codec getCodec() {- return new Lucene103Codec() {- @Override- public KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswScalarQuantizedVectorsFormat(+ return TestUtil.alwaysKnnVectorsFormat(+ new Lucene99HnswScalarQuantizedVectorsFormat(Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,- Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH);- }- };+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH));}@Overridediff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.javaindex c72809da847..6139b375947 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java@@ -21,9 +21,8 @@ import java.nio.file.Files;import java.nio.file.Path;import java.util.Random;import java.util.concurrent.TimeUnit;-import org.apache.lucene.codecs.lucene103.ForDeltaUtil;-import org.apache.lucene.codecs.lucene103.ForUtil;-import org.apache.lucene.codecs.lucene103.PostingIndexInput;+import org.apache.lucene.codecs.lucene104.ForUtil;+import org.apache.lucene.codecs.lucene104.PostingIndexInput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;@@ -60,7 +59,6 @@ public class PostingIndexInputBenchmark {private IndexInput in;private PostingIndexInput postingIn;private final ForUtil forUtil = new ForUtil();- private final ForDeltaUtil forDeltaUtil = new ForDeltaUtil();private final int[] values = new int[ForUtil.BLOCK_SIZE];@Param({"2", "3", "4", "5", "6", "7", "8", "9", "10"})@@ -78,7 +76,7 @@ public class PostingIndexInputBenchmark {}}in = dir.openInput("docs", IOContext.DEFAULT);- postingIn = new PostingIndexInput(in, forUtil, forDeltaUtil);+ postingIn = new PostingIndexInput(in, forUtil);}@TearDown(Level.Trial)@@ -99,13 +97,6 @@ public class PostingIndexInputBenchmark {bh.consume(values);}- @Benchmark- public void decodeAndPrefixSum(Blackhole bh) throws IOException {- in.seek(3); // random unaligned offset- postingIn.decodeAndPrefixSum(bpv, 100, values);- bh.consume(values);- }-@Benchmark@Fork(jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})public void decodeVector(Blackhole bh) throws IOException {@@ -113,12 +104,4 @@ public class PostingIndexInputBenchmark {postingIn.decode(bpv, values);bh.consume(values);}-- @Benchmark- @Fork(jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})- public void decodeAndPrefixSumVector(Blackhole bh) throws IOException {- in.seek(3); // random unaligned offset- postingIn.decodeAndPrefixSum(bpv, 100, values);- bh.consume(values);- }}diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.javaindex f27908bf1e9..36195f66012 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java@@ -22,14 +22,14 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.util.IOUtils;-/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene103PostingsWriter}. */+/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene104PostingsWriter}. */public class BlockTreeOrdsPostingsFormat extends PostingsFormat {private final int minTermBlockSize;@@ -67,7 +67,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);try {return new OrdsBlockTreeTermsWriter(@@ -80,7 +80,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);try {return new OrdsBlockTreeTermsReader(postingsReader, state);} catch (Throwable t) {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.javaindex 6a14c85430d..cbf6c7e660f 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java@@ -24,7 +24,7 @@ import java.util.TreeMap;import org.apache.lucene.codecs.FieldsConsumer;import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;import org.apache.lucene.index.BaseTermsEnum;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.Fields;@@ -54,7 +54,7 @@ import org.apache.lucene.util.automaton.TransitionAccessor;// - or: longer dense skip lists than just next byte?/**- * Wraps {@link Lucene103PostingsFormat} format for on-disk storage, but then at read time loads and+ * Wraps {@link Lucene104PostingsFormat} format for on-disk storage, but then at read time loads and* stores all terms and postings directly in RAM as byte[], int[].** <p><b>WARNING</b>: This is exceptionally RAM intensive: it makes no effort to compress the@@ -97,12 +97,12 @@ public final class DirectPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- return PostingsFormat.forName("Lucene103").fieldsConsumer(state);+ return PostingsFormat.forName("Lucene104").fieldsConsumer(state);}@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- FieldsProducer postings = PostingsFormat.forName("Lucene103").fieldsProducer(state);+ FieldsProducer postings = PostingsFormat.forName("Lucene104").fieldsProducer(state);if (state.context.context() != IOContext.Context.MERGE) {FieldsProducer loadedPostings;try {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.javaindex e535ccd33db..a72fde421d6 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java@@ -22,8 +22,8 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.util.IOUtils;@@ -41,7 +41,7 @@ public final class FSTPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);try {return new FSTTermsWriter(state, postingsWriter);@@ -53,7 +53,7 @@ public final class FSTPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);try {return new FSTTermsReader(state, postingsReader);} catch (Throwable t) {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.javaindex a4a9048a73e..af15ab034ee 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java@@ -17,13 +17,13 @@package org.apache.lucene.codecs.uniformsplit;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.BLOCK_SIZE;import java.io.IOException;import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.TermState;@@ -34,7 +34,7 @@ import org.apache.lucene.util.RamUsageEstimator;/*** {@link TermState} serializer which encodes each file pointer as a delta relative to a base file- * pointer. It differs from {@link Lucene103PostingsWriter#encodeTerm} which encodes each file+ * pointer. It differs from {@link Lucene104PostingsWriter#encodeTerm} which encodes each file* pointer as a delta relative to the previous file pointer.** <p>It automatically sets the base file pointer to the first valid file pointer for doc start FP,@@ -95,7 +95,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {/*** Writes a {@link BlockTermState} to the provided {@link DataOutput}.*- * <p>Simpler variant of {@link Lucene103PostingsWriter#encodeTerm(DataOutput, FieldInfo,+ * <p>Simpler variant of {@link Lucene104PostingsWriter#encodeTerm(DataOutput, FieldInfo,* BlockTermState, boolean)}.*/public void writeTermState(@@ -145,7 +145,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {/*** Reads a {@link BlockTermState} from the provided {@link DataInput}.*- * <p>Simpler variant of {@link Lucene103PostingsReader#decodeTerm(DataInput, FieldInfo,+ * <p>Simpler variant of {@link Lucene104PostingsReader#decodeTerm(DataInput, FieldInfo,* BlockTermState, boolean)}.** @param reuse {@link BlockTermState} to reuse; or null to create a new one.diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.javaindex 64c3a4f5609..801143ae232 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.util.IOUtils;@@ -113,7 +113,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);try {return createUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);@@ -125,7 +125,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);try {return createUniformSplitTermsReader(postingsReader, state, blockDecoder);} catch (Throwable t) {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.javaindex a4ac5603913..fedeb0ecf52 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java@@ -28,7 +28,7 @@* org.apache.lucene.search.PhraseQuery})* <li>Quite efficient for {@link org.apache.lucene.search.PrefixQuery}* <li>Not efficient for spell-check and {@link org.apache.lucene.search.FuzzyQuery}, in this case- * prefer {@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat}+ * prefer {@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat}* </ul>*/package org.apache.lucene.codecs.uniformsplit;diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.javaindex 2f6dc6fa749..66a923dc0b4 100644--- a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java@@ -17,7 +17,7 @@package org.apache.lucene.codecs.lucene90.tests;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;/** Test utility class to create mock {@link IntBlockTermState}. */public class MockTermStateFactory {diff --git a/lucene/core/src/generated/checksums/generateForUtil.json b/lucene/core/src/generated/checksums/generateForUtil.jsonindex 8a50acc81d5..e7970804e68 100644--- a/lucene/core/src/generated/checksums/generateForUtil.json+++ b/lucene/core/src/generated/checksums/generateForUtil.json@@ -1,4 +1,4 @@{- "lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java": "1ea2dac2a26be521a70cf74c37fe9134f7b42cb7",- "lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py": "cd186fe09ceeedb4e2dd3be191d7461e67fa1c2b"+ "lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java": "5dda079c68e6060217f29010618c7fd807583056",+ "lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py": "4692fed62d9f79554647c5423b96b9e60c9f30eb"}diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.javaindex b9030b52a47..17876735f1f 100644--- a/lucene/core/src/java/module-info.java+++ b/lucene/core/src/java/module-info.java@@ -32,7 +32,7 @@ module org.apache.lucene.core {exports org.apache.lucene.codecs.lucene99;exports org.apache.lucene.codecs.lucene102;exports org.apache.lucene.codecs.lucene103.blocktree;- exports org.apache.lucene.codecs.lucene103;+ exports org.apache.lucene.codecs.lucene104;exports org.apache.lucene.codecs.perfield;exports org.apache.lucene.codecs;exports org.apache.lucene.document;@@ -79,7 +79,7 @@ module org.apache.lucene.core {provides org.apache.lucene.analysis.TokenizerFactory withorg.apache.lucene.analysis.standard.StandardTokenizerFactory;provides org.apache.lucene.codecs.Codec with- org.apache.lucene.codecs.lucene103.Lucene103Codec;+ org.apache.lucene.codecs.lucene104.Lucene104Codec;provides org.apache.lucene.codecs.DocValuesFormat withorg.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;provides org.apache.lucene.codecs.KnnVectorsFormat with@@ -89,7 +89,7 @@ module org.apache.lucene.core {org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat,org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat;provides org.apache.lucene.codecs.PostingsFormat with- org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;+ org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;provides org.apache.lucene.index.SortFieldProvider withorg.apache.lucene.search.SortField.Provider,org.apache.lucene.search.SortedNumericSortField.Provider,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.javaindex 026c3b7b36c..b35d7a3f0bb 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java@@ -56,7 +56,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {}@SuppressWarnings("NonFinalStaticField")- static Codec defaultCodec = LOADER.lookup("Lucene103");+ static Codec defaultCodec = LOADER.lookup("Lucene104");}private final String name;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.javaindex 05fbb64cd83..335a8d24813 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java@@ -151,15 +151,15 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term dictionary}. A+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term dictionary}. A* dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Frequency data}. For+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Frequency data}. For* each term in the dictionary, the numbers of all the documents that contain that term, and* the frequency of the term in that document, unless frequencies are omitted ({@link* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Proximity data}. For+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Proximity data}. For* each term in the dictionary, the positions that the term occurs in each document. Note that* this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/BitSetUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/BitSetUtil.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/BitSetUtil.javarename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/BitSetUtil.javaindex da1b90fd39f..d5d72d35d84 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/BitSetUtil.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/BitSetUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;import java.util.Objects;import org.apache.lucene.util.FixedBitSet;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.javanew file mode 100644index 00000000000..3a2ad7396c7--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java@@ -0,0 +1,535 @@+// This file has been automatically generated, DO NOT EDIT++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;+import org.apache.lucene.store.DataOutput;++/**+ * Inspired from https://fulmicoton.com/posts/bitpacking/ Encodes multiple integers in one to get+ * SIMD-like speedups. If bitsPerValue <= 8 then we pack 4 ints per Java int else if bitsPerValue+ * <= 16 we pack 2 ints per Java int else we do scalar operations.+ */+public final class ForUtil {++ /** Number of integers per block. */+ public static final int BLOCK_SIZE = 256;++ static final int BLOCK_SIZE_LOG2 = 8;++ static int expandMask16(int mask16) {+ return mask16 | (mask16 << 16);+ }++ static int expandMask8(int mask8) {+ return expandMask16(mask8 | (mask8 << 8));+ }++ static int mask32(int bitsPerValue) {+ return (1 << bitsPerValue) - 1;+ }++ static int mask16(int bitsPerValue) {+ return expandMask16((1 << bitsPerValue) - 1);+ }++ static int mask8(int bitsPerValue) {+ return expandMask8((1 << bitsPerValue) - 1);+ }++ static void expand8(int[] arr) {+ for (int i = 0; i < 64; ++i) {+ int l = arr[i];+ arr[i] = (l >>> 24) & 0xFF;+ arr[64 + i] = (l >>> 16) & 0xFF;+ arr[128 + i] = (l >>> 8) & 0xFF;+ arr[192 + i] = l & 0xFF;+ }+ }++ static void collapse8(int[] arr) {+ for (int i = 0; i < 64; ++i) {+ arr[i] = (arr[i] << 24) | (arr[64 + i] << 16) | (arr[128 + i] << 8) | arr[192 + i];+ }+ }++ static void expand16(int[] arr) {+ for (int i = 0; i < 128; ++i) {+ int l = arr[i];+ arr[i] = (l >>> 16) & 0xFFFF;+ arr[128 + i] = l & 0xFFFF;+ }+ }++ static void collapse16(int[] arr) {+ for (int i = 0; i < 128; ++i) {+ arr[i] = (arr[i] << 16) | arr[128 + i];+ }+ }++ private final int[] tmp = new int[BLOCK_SIZE];++ /** Encode 256 integers from {@code ints} into {@code out}. */+ void encode(int[] ints, int bitsPerValue, DataOutput out) throws IOException {+ final int nextPrimitive;+ if (bitsPerValue <= 8) {+ nextPrimitive = 8;+ collapse8(ints);+ } else if (bitsPerValue <= 16) {+ nextPrimitive = 16;+ collapse16(ints);+ } else {+ nextPrimitive = 32;+ }+ encode(ints, bitsPerValue, nextPrimitive, out, tmp);+ }++ static void encode(int[] ints, int bitsPerValue, int primitiveSize, DataOutput out, int[] tmp)+ throws IOException {+ final int numInts = BLOCK_SIZE * primitiveSize / Integer.SIZE;++ final int numIntsPerShift = bitsPerValue * 8;+ int idx = 0;+ int shift = primitiveSize - bitsPerValue;+ for (int i = 0; i < numIntsPerShift; ++i) {+ tmp[i] = ints[idx++] << shift;+ }+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {+ for (int i = 0; i < numIntsPerShift; ++i) {+ tmp[i] |= ints[idx++] << shift;+ }+ }++ final int remainingBitsPerInt = shift + bitsPerValue;+ final int maskRemainingBitsPerInt;+ if (primitiveSize == 8) {+ maskRemainingBitsPerInt = MASKS8[remainingBitsPerInt];+ } else if (primitiveSize == 16) {+ maskRemainingBitsPerInt = MASKS16[remainingBitsPerInt];+ } else {+ maskRemainingBitsPerInt = MASKS32[remainingBitsPerInt];+ }++ int tmpIdx = 0;+ int remainingBitsPerValue = bitsPerValue;+ while (idx < numInts) {+ if (remainingBitsPerValue >= remainingBitsPerInt) {+ remainingBitsPerValue -= remainingBitsPerInt;+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerInt;+ if (remainingBitsPerValue == 0) {+ idx++;+ remainingBitsPerValue = bitsPerValue;+ }+ } else {+ final int mask1, mask2;+ if (primitiveSize == 8) {+ mask1 = MASKS8[remainingBitsPerValue];+ mask2 = MASKS8[remainingBitsPerInt - remainingBitsPerValue];+ } else if (primitiveSize == 16) {+ mask1 = MASKS16[remainingBitsPerValue];+ mask2 = MASKS16[remainingBitsPerInt - remainingBitsPerValue];+ } else {+ mask1 = MASKS32[remainingBitsPerValue];+ mask2 = MASKS32[remainingBitsPerInt - remainingBitsPerValue];+ }+ tmp[tmpIdx] |= (ints[idx++] & mask1) << (remainingBitsPerInt - remainingBitsPerValue);+ remainingBitsPerValue = bitsPerValue - remainingBitsPerInt + remainingBitsPerValue;+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & mask2;+ }+ }++ for (int i = 0; i < numIntsPerShift; ++i) {+ out.writeInt(tmp[i]);+ }+ }++ /** Number of bytes required to encode 256 integers of {@code bitsPerValue} bits per value. */+ static int numBytes(int bitsPerValue) {+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);+ }++ static void decodeSlow(int bitsPerValue, PostingDecodingUtil pdu, int[] tmp, int[] ints)+ throws IOException {+ final int numInts = bitsPerValue << 3;+ final int mask = MASKS32[bitsPerValue];+ pdu.splitInts(numInts, ints, 32 - bitsPerValue, 32, mask, tmp, 0, -1);+ final int remainingBitsPerInt = 32 - bitsPerValue;+ final int mask32RemainingBitsPerInt = MASKS32[remainingBitsPerInt];+ int tmpIdx = 0;+ int remainingBits = remainingBitsPerInt;+ for (int intsIdx = numInts; intsIdx < BLOCK_SIZE; ++intsIdx) {+ int b = bitsPerValue - remainingBits;+ int l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;+ while (b >= remainingBitsPerInt) {+ b -= remainingBitsPerInt;+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerInt) << b;+ }+ if (b > 0) {+ l |= (tmp[tmpIdx] >>> (remainingBitsPerInt - b)) & MASKS32[b];+ remainingBits = remainingBitsPerInt - b;+ } else {+ remainingBits = remainingBitsPerInt;+ }+ ints[intsIdx] = l;+ }+ }++ static final int[] MASKS8 = new int[8];+ static final int[] MASKS16 = new int[16];+ static final int[] MASKS32 = new int[32];++ static {+ for (int i = 0; i < 8; ++i) {+ MASKS8[i] = mask8(i);+ }+ for (int i = 0; i < 16; ++i) {+ MASKS16[i] = mask16(i);+ }+ for (int i = 0; i < 32; ++i) {+ MASKS32[i] = mask32(i);+ }+ }++ // mark values in array as final ints to avoid the cost of reading array, arrays should only be+ // used when the idx is a variable+ static final int MASK8_1 = MASKS8[1];+ static final int MASK8_2 = MASKS8[2];+ static final int MASK8_3 = MASKS8[3];+ static final int MASK8_4 = MASKS8[4];+ static final int MASK8_5 = MASKS8[5];+ static final int MASK8_6 = MASKS8[6];+ static final int MASK8_7 = MASKS8[7];+ static final int MASK16_1 = MASKS16[1];+ static final int MASK16_2 = MASKS16[2];+ static final int MASK16_3 = MASKS16[3];+ static final int MASK16_4 = MASKS16[4];+ static final int MASK16_5 = MASKS16[5];+ static final int MASK16_6 = MASKS16[6];+ static final int MASK16_7 = MASKS16[7];+ static final int MASK16_8 = MASKS16[8];+ static final int MASK16_9 = MASKS16[9];+ static final int MASK16_10 = MASKS16[10];+ static final int MASK16_11 = MASKS16[11];+ static final int MASK16_12 = MASKS16[12];+ static final int MASK16_13 = MASKS16[13];+ static final int MASK16_14 = MASKS16[14];+ static final int MASK16_15 = MASKS16[15];+ static final int MASK32_1 = MASKS32[1];+ static final int MASK32_2 = MASKS32[2];+ static final int MASK32_3 = MASKS32[3];+ static final int MASK32_4 = MASKS32[4];+ static final int MASK32_5 = MASKS32[5];+ static final int MASK32_6 = MASKS32[6];+ static final int MASK32_7 = MASKS32[7];+ static final int MASK32_8 = MASKS32[8];+ static final int MASK32_9 = MASKS32[9];+ static final int MASK32_10 = MASKS32[10];+ static final int MASK32_11 = MASKS32[11];+ static final int MASK32_12 = MASKS32[12];+ static final int MASK32_13 = MASKS32[13];+ static final int MASK32_14 = MASKS32[14];+ static final int MASK32_15 = MASKS32[15];+ static final int MASK32_16 = MASKS32[16];++ /** Decode 256 integers into {@code ints}. */+ void decode(int bitsPerValue, PostingDecodingUtil pdu, int[] ints) throws IOException {+ switch (bitsPerValue) {+ case 1:+ decode1(pdu, ints);+ expand8(ints);+ break;+ case 2:+ decode2(pdu, ints);+ expand8(ints);+ break;+ case 3:+ decode3(pdu, tmp, ints);+ expand8(ints);+ break;+ case 4:+ decode4(pdu, ints);+ expand8(ints);+ break;+ case 5:+ decode5(pdu, tmp, ints);+ expand8(ints);+ break;+ case 6:+ decode6(pdu, tmp, ints);+ expand8(ints);+ break;+ case 7:+ decode7(pdu, tmp, ints);+ expand8(ints);+ break;+ case 8:+ decode8(pdu, ints);+ expand8(ints);+ break;+ case 9:+ decode9(pdu, tmp, ints);+ expand16(ints);+ break;+ case 10:+ decode10(pdu, tmp, ints);+ expand16(ints);+ break;+ case 11:+ decode11(pdu, tmp, ints);+ expand16(ints);+ break;+ case 12:+ decode12(pdu, tmp, ints);+ expand16(ints);+ break;+ case 13:+ decode13(pdu, tmp, ints);+ expand16(ints);+ break;+ case 14:+ decode14(pdu, tmp, ints);+ expand16(ints);+ break;+ case 15:+ decode15(pdu, tmp, ints);+ expand16(ints);+ break;+ case 16:+ decode16(pdu, ints);+ expand16(ints);+ break;+ default:+ decodeSlow(bitsPerValue, pdu, tmp, ints);+ break;+ }+ }++ static void decode1(PostingDecodingUtil pdu, int[] ints) throws IOException {+ pdu.splitInts(8, ints, 7, 1, MASK8_1, ints, 56, MASK8_1);+ }++ static void decode2(PostingDecodingUtil pdu, int[] ints) throws IOException {+ pdu.splitInts(16, ints, 6, 2, MASK8_2, ints, 48, MASK8_2);+ }++ static void decode3(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(24, ints, 5, 3, MASK8_3, tmp, 0, MASK8_2);+ for (int iter = 0, tmpIdx = 0, intsIdx = 48; iter < 8; ++iter, tmpIdx += 3, intsIdx += 2) {+ int l0 = tmp[tmpIdx + 0] << 1;+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;+ ints[intsIdx + 0] = l0;+ int l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;+ l1 |= tmp[tmpIdx + 2] << 0;+ ints[intsIdx + 1] = l1;+ }+ }++ static void decode4(PostingDecodingUtil pdu, int[] ints) throws IOException {+ pdu.splitInts(32, ints, 4, 4, MASK8_4, ints, 32, MASK8_4);+ }++ static void decode5(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(40, ints, 3, 5, MASK8_5, tmp, 0, MASK8_3);+ for (int iter = 0, tmpIdx = 0, intsIdx = 40; iter < 8; ++iter, tmpIdx += 5, intsIdx += 3) {+ int l0 = tmp[tmpIdx + 0] << 2;+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;+ ints[intsIdx + 0] = l0;+ int l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;+ l1 |= tmp[tmpIdx + 2] << 1;+ l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;+ ints[intsIdx + 1] = l1;+ int l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;+ l2 |= tmp[tmpIdx + 4] << 0;+ ints[intsIdx + 2] = l2;+ }+ }++ static void decode6(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(48, ints, 2, 6, MASK8_6, tmp, 0, MASK8_2);+ for (int iter = 0, tmpIdx = 0, intsIdx = 48; iter < 16; ++iter, tmpIdx += 3, intsIdx += 1) {+ int l0 = tmp[tmpIdx + 0] << 4;+ l0 |= tmp[tmpIdx + 1] << 2;+ l0 |= tmp[tmpIdx + 2] << 0;+ ints[intsIdx + 0] = l0;+ }+ }++ static void decode7(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(56, ints, 1, 7, MASK8_7, tmp, 0, MASK8_1);+ for (int iter = 0, tmpIdx = 0, intsIdx = 56; iter < 8; ++iter, tmpIdx += 7, intsIdx += 1) {+ int l0 = tmp[tmpIdx + 0] << 6;+ l0 |= tmp[tmpIdx + 1] << 5;+ l0 |= tmp[tmpIdx + 2] << 4;+ l0 |= tmp[tmpIdx + 3] << 3;+ l0 |= tmp[tmpIdx + 4] << 2;+ l0 |= tmp[tmpIdx + 5] << 1;+ l0 |= tmp[tmpIdx + 6] << 0;+ ints[intsIdx + 0] = l0;+ }+ }++ static void decode8(PostingDecodingUtil pdu, int[] ints) throws IOException {+ pdu.in.readInts(ints, 0, 64);+ }++ static void decode9(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(72, ints, 7, 9, MASK16_9, tmp, 0, MASK16_7);+ for (int iter = 0, tmpIdx = 0, intsIdx = 72; iter < 8; ++iter, tmpIdx += 9, intsIdx += 7) {+ int l0 = tmp[tmpIdx + 0] << 2;+ l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;+ ints[intsIdx + 0] = l0;+ int l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;+ l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;+ ints[intsIdx + 1] = l1;+ int l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;+ l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;+ ints[intsIdx + 2] = l2;+ int l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;+ l3 |= tmp[tmpIdx + 4] << 1;+ l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;+ ints[intsIdx + 3] = l3;+ int l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;+ l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;+ ints[intsIdx + 4] = l4;+ int l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;+ l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;+ ints[intsIdx + 5] = l5;+ int l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;+ l6 |= tmp[tmpIdx + 8] << 0;+ ints[intsIdx + 6] = l6;+ }+ }++ static void decode10(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(80, ints, 6, 10, MASK16_10, tmp, 0, MASK16_6);+ for (int iter = 0, tmpIdx = 0, intsIdx = 80; iter < 16; ++iter, tmpIdx += 5, intsIdx += 3) {+ int l0 = tmp[tmpIdx + 0] << 4;+ l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;+ ints[intsIdx + 0] = l0;+ int l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;+ l1 |= tmp[tmpIdx + 2] << 2;+ l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;+ ints[intsIdx + 1] = l1;+ int l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;+ l2 |= tmp[tmpIdx + 4] << 0;+ ints[intsIdx + 2] = l2;+ }+ }++ static void decode11(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(88, ints, 5, 11, MASK16_11, tmp, 0, MASK16_5);+ for (int iter = 0, tmpIdx = 0, intsIdx = 88; iter < 8; ++iter, tmpIdx += 11, intsIdx += 5) {+ int l0 = tmp[tmpIdx + 0] << 6;+ l0 |= tmp[tmpIdx + 1] << 1;+ l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;+ ints[intsIdx + 0] = l0;+ int l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;+ l1 |= tmp[tmpIdx + 3] << 2;+ l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;+ ints[intsIdx + 1] = l1;+ int l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;+ l2 |= tmp[tmpIdx + 5] << 3;+ l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;+ ints[intsIdx + 2] = l2;+ int l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;+ l3 |= tmp[tmpIdx + 7] << 4;+ l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;+ ints[intsIdx + 3] = l3;+ int l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;+ l4 |= tmp[tmpIdx + 9] << 5;+ l4 |= tmp[tmpIdx + 10] << 0;+ ints[intsIdx + 4] = l4;+ }+ }++ static void decode12(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(96, ints, 4, 12, MASK16_12, tmp, 0, MASK16_4);+ for (int iter = 0, tmpIdx = 0, intsIdx = 96; iter < 32; ++iter, tmpIdx += 3, intsIdx += 1) {+ int l0 = tmp[tmpIdx + 0] << 8;+ l0 |= tmp[tmpIdx + 1] << 4;+ l0 |= tmp[tmpIdx + 2] << 0;+ ints[intsIdx + 0] = l0;+ }+ }++ static void decode13(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(104, ints, 3, 13, MASK16_13, tmp, 0, MASK16_3);+ for (int iter = 0, tmpIdx = 0, intsIdx = 104; iter < 8; ++iter, tmpIdx += 13, intsIdx += 3) {+ int l0 = tmp[tmpIdx + 0] << 10;+ l0 |= tmp[tmpIdx + 1] << 7;+ l0 |= tmp[tmpIdx + 2] << 4;+ l0 |= tmp[tmpIdx + 3] << 1;+ l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;+ ints[intsIdx + 0] = l0;+ int l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;+ l1 |= tmp[tmpIdx + 5] << 8;+ l1 |= tmp[tmpIdx + 6] << 5;+ l1 |= tmp[tmpIdx + 7] << 2;+ l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;+ ints[intsIdx + 1] = l1;+ int l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;+ l2 |= tmp[tmpIdx + 9] << 9;+ l2 |= tmp[tmpIdx + 10] << 6;+ l2 |= tmp[tmpIdx + 11] << 3;+ l2 |= tmp[tmpIdx + 12] << 0;+ ints[intsIdx + 2] = l2;+ }+ }++ static void decode14(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(112, ints, 2, 14, MASK16_14, tmp, 0, MASK16_2);+ for (int iter = 0, tmpIdx = 0, intsIdx = 112; iter < 16; ++iter, tmpIdx += 7, intsIdx += 1) {+ int l0 = tmp[tmpIdx + 0] << 12;+ l0 |= tmp[tmpIdx + 1] << 10;+ l0 |= tmp[tmpIdx + 2] << 8;+ l0 |= tmp[tmpIdx + 3] << 6;+ l0 |= tmp[tmpIdx + 4] << 4;+ l0 |= tmp[tmpIdx + 5] << 2;+ l0 |= tmp[tmpIdx + 6] << 0;+ ints[intsIdx + 0] = l0;+ }+ }++ static void decode15(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {+ pdu.splitInts(120, ints, 1, 15, MASK16_15, tmp, 0, MASK16_1);+ for (int iter = 0, tmpIdx = 0, intsIdx = 120; iter < 8; ++iter, tmpIdx += 15, intsIdx += 1) {+ int l0 = tmp[tmpIdx + 0] << 14;+ l0 |= tmp[tmpIdx + 1] << 13;+ l0 |= tmp[tmpIdx + 2] << 12;+ l0 |= tmp[tmpIdx + 3] << 11;+ l0 |= tmp[tmpIdx + 4] << 10;+ l0 |= tmp[tmpIdx + 5] << 9;+ l0 |= tmp[tmpIdx + 6] << 8;+ l0 |= tmp[tmpIdx + 7] << 7;+ l0 |= tmp[tmpIdx + 8] << 6;+ l0 |= tmp[tmpIdx + 9] << 5;+ l0 |= tmp[tmpIdx + 10] << 4;+ l0 |= tmp[tmpIdx + 11] << 3;+ l0 |= tmp[tmpIdx + 12] << 2;+ l0 |= tmp[tmpIdx + 13] << 1;+ l0 |= tmp[tmpIdx + 14] << 0;+ ints[intsIdx + 0] = l0;+ }+ }++ static void decode16(PostingDecodingUtil pdu, int[] ints) throws IOException {+ pdu.in.readInts(ints, 0, 128);+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104Codec.javanew file mode 100644index 00000000000..f2b7317bc93--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104Codec.java@@ -0,0 +1,217 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.util.Objects;+import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.CompoundFormat;+import org.apache.lucene.codecs.DocValuesFormat;+import org.apache.lucene.codecs.FieldInfosFormat;+import org.apache.lucene.codecs.FilterCodec;+import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.LiveDocsFormat;+import org.apache.lucene.codecs.NormsFormat;+import org.apache.lucene.codecs.PointsFormat;+import org.apache.lucene.codecs.PostingsFormat;+import org.apache.lucene.codecs.SegmentInfoFormat;+import org.apache.lucene.codecs.StoredFieldsFormat;+import org.apache.lucene.codecs.TermVectorsFormat;+import org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat;+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;+import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;+import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;+import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;+import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;+import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;+import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat;+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;+import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;++/**+ * Implements the Lucene 10.4 index format+ *+ * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.+ *+ * @see org.apache.lucene.codecs.lucene104 package documentation for file format details.+ * @lucene.experimental+ */+public class Lucene104Codec extends Codec {++ /** Configuration option for the codec. */+ public enum Mode {+ /** Trade compression ratio for retrieval speed. */+ BEST_SPEED(Lucene90StoredFieldsFormat.Mode.BEST_SPEED),+ /** Trade retrieval speed for compression ratio. */+ BEST_COMPRESSION(Lucene90StoredFieldsFormat.Mode.BEST_COMPRESSION);++ private final Lucene90StoredFieldsFormat.Mode storedMode;++ private Mode(Lucene90StoredFieldsFormat.Mode storedMode) {+ this.storedMode = Objects.requireNonNull(storedMode);+ }+ }++ private final TermVectorsFormat vectorsFormat = new Lucene90TermVectorsFormat();+ private final FieldInfosFormat fieldInfosFormat = new Lucene94FieldInfosFormat();+ private final SegmentInfoFormat segmentInfosFormat = new Lucene99SegmentInfoFormat();+ private final LiveDocsFormat liveDocsFormat = new Lucene90LiveDocsFormat();+ private final CompoundFormat compoundFormat = new Lucene90CompoundFormat();+ private final NormsFormat normsFormat = new Lucene90NormsFormat();++ private final PostingsFormat defaultPostingsFormat;+ private final PostingsFormat postingsFormat =+ new PerFieldPostingsFormat() {+ @Override+ public PostingsFormat getPostingsFormatForField(String field) {+ return Lucene104Codec.this.getPostingsFormatForField(field);+ }+ };++ private final DocValuesFormat defaultDVFormat;+ private final DocValuesFormat docValuesFormat =+ new PerFieldDocValuesFormat() {+ @Override+ public DocValuesFormat getDocValuesFormatForField(String field) {+ return Lucene104Codec.this.getDocValuesFormatForField(field);+ }+ };++ private final KnnVectorsFormat defaultKnnVectorsFormat;+ private final KnnVectorsFormat knnVectorsFormat =+ new PerFieldKnnVectorsFormat() {+ @Override+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {+ return Lucene104Codec.this.getKnnVectorsFormatForField(field);+ }+ };++ private final StoredFieldsFormat storedFieldsFormat;++ /** Instantiates a new codec. */+ public Lucene104Codec() {+ this(Mode.BEST_SPEED);+ }++ /**+ * Instantiates a new codec, specifying the stored fields compression mode to use.+ *+ * @param mode stored fields compression mode to use for newly flushed/merged segments.+ */+ public Lucene104Codec(Mode mode) {+ super("Lucene104");+ this.storedFieldsFormat =+ new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);+ this.defaultPostingsFormat = new Lucene104PostingsFormat();+ this.defaultDVFormat = new Lucene90DocValuesFormat();+ this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();+ }++ @Override+ public final StoredFieldsFormat storedFieldsFormat() {+ return storedFieldsFormat;+ }++ @Override+ public final TermVectorsFormat termVectorsFormat() {+ return vectorsFormat;+ }++ @Override+ public final PostingsFormat postingsFormat() {+ return postingsFormat;+ }++ @Override+ public final FieldInfosFormat fieldInfosFormat() {+ return fieldInfosFormat;+ }++ @Override+ public final SegmentInfoFormat segmentInfoFormat() {+ return segmentInfosFormat;+ }++ @Override+ public final LiveDocsFormat liveDocsFormat() {+ return liveDocsFormat;+ }++ @Override+ public final CompoundFormat compoundFormat() {+ return compoundFormat;+ }++ @Override+ public final PointsFormat pointsFormat() {+ return new Lucene90PointsFormat();+ }++ @Override+ public final KnnVectorsFormat knnVectorsFormat() {+ return knnVectorsFormat;+ }++ /**+ * Returns the postings format that should be used for writing new segments of <code>field</code>.+ *+ * <p>The default implementation always returns "Lucene104".+ *+ * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:+ * future version of Lucene are only guaranteed to be able to read the default implementation,+ */+ public PostingsFormat getPostingsFormatForField(String field) {+ return defaultPostingsFormat;+ }++ /**+ * Returns the docvalues format that should be used for writing new segments of <code>field</code>+ * .+ *+ * <p>The default implementation always returns "Lucene90".+ *+ * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:+ * future version of Lucene are only guaranteed to be able to read the default implementation.+ */+ public DocValuesFormat getDocValuesFormatForField(String field) {+ return defaultDVFormat;+ }++ /**+ * Returns the vectors format that should be used for writing new segments of <code>field</code>+ *+ * <p>The default implementation always returns "Lucene99HnswVectorsFormat".+ *+ * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:+ * future version of Lucene are only guaranteed to be able to read the default implementation.+ */+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {+ return defaultKnnVectorsFormat;+ }++ @Override+ public final DocValuesFormat docValuesFormat() {+ return docValuesFormat;+ }++ @Override+ public final NormsFormat normsFormat() {+ return normsFormat;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsFormat.javanew file mode 100644index 00000000000..b4828145aba--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsFormat.java@@ -0,0 +1,505 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.FieldsConsumer;+import org.apache.lucene.codecs.FieldsProducer;+import org.apache.lucene.codecs.PostingsFormat;+import org.apache.lucene.codecs.PostingsReaderBase;+import org.apache.lucene.codecs.PostingsWriterBase;+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;+import org.apache.lucene.index.ImpactsEnum;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.index.TermState;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.packed.PackedInts;++/**+ * Lucene 10.4 postings format, which encodes postings in packed integer blocks for fast decode.+ *+ * <p>Basic idea:+ *+ * <ul>+ * <li><b>Packed Blocks and VInt Blocks</b>:+ * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed+ * format}): the block size (i.e. number of integers inside block) is fixed (currently 256).+ * Additionally blocks that are all the same value are encoded in an optimized way.+ * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block+ * size is variable.+ * <li><b>Block structure</b>:+ * <p>When the postings are long enough, Lucene104PostingsFormat will try to encode most+ * integer data as a packed block.+ * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as+ * two packed blocks, while the remaining 3 are encoded as one VInt block.+ * <p>Different kinds of data are always encoded separately into different packed blocks, but+ * may possibly be interleaved into the same VInt block.+ * <p>This strategy is applied to pairs: <document number, frequency>, <position,+ * payload length>, <position, offset start, offset length>, and <position,+ * payload length, offsetstart, offset length>.+ * <li><b>Skipdata</b>:+ * <p>Skipdata is interleaved with blocks on 2 levels. Level 0 skip data is interleaved+ * between every packed block. Level 1 skip data is interleaved between every 32 packed+ * blocks.+ * <li><b>Positions, Payloads, and Offsets</b>:+ * <p>A position is an integer indicating where the term occurs within one document. A payload+ * is a blob of metadata associated with current position. An offset is a pair of integers+ * indicating the tokenized start/end offsets for given term in current position: it is+ * essentially a specialized payload.+ * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets+ * (assuming a null payload contributes one count). As mentioned in block structure, it is+ * possible to encode these three either combined or separately.+ * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,+ * position data is separated out as .pos, while payloads and offsets are encoded in .pay+ * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all+ * these three are stored interleaved into the .pos (so is payload metadata).+ * <p>With this strategy, the majority of payload and offset data will be outside .pos file.+ * So for queries that require only position data, running on a full index with payloads and+ * offsets, this reduces disk pre-fetches.+ * </ul>+ *+ * <p>Files and detailed format:+ *+ * <ul>+ * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>+ * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>+ * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>+ * <li><code>.pos</code>: <a href="#Positions">Positions</a>+ * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>+ * </ul>+ *+ * <a id="Termdictionary"></a>+ *+ * <dl>+ * <dd><b>Term Dictionary</b>+ * <p>The .tim file contains the list of terms in each field along with per-term statistics+ * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the+ * .doc, .pos, and .pay files. See {@link Lucene103BlockTreeTermsWriter} for more details on+ * the format.+ * <p>NOTE: The term dictionary can plug into different postings implementations: the postings+ * writer/reader are actually responsible for encoding and decoding the PostingsHeader and+ * TermMetadata sections described here:+ * <ul>+ * <li>PostingsHeader --> Header, PackedBlockSize+ * <li>TermMetadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,+ * PayFPDelta?+ * <li>Header, --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}+ * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta --> {@link+ * DataOutput#writeVLong VLong}+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version+ * information for the postings.+ * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width+ * is determined by the largest integer. Smaller block size result in smaller variance+ * among width of integers hence smaller indexes. Larger block size result in more+ * efficient bulk i/o hence better acceleration. This value should always be a multiple+ * of 64, currently fixed as 256 as a tradeoff. It is also the skip interval used to+ * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.+ * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In+ * particular, it is the difference of file offset between this term's data and previous+ * term's data (or zero, for the first term in the block).On disk it is stored as the+ * difference from previous value in sequence.+ * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.+ * While PayFPDelta determines the position of this term's <TermPayloads,+ * TermOffsets?> within the .pay file. Similar to DocFPDelta, it is the difference+ * between two file positions (or neglected, for fields that omit payloads and offsets).+ * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last+ * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or+ * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to+ * load following payloads and offsets from .pos instead of .pay. Every time a new block+ * of positions are to be loaded, the PostingsReader will use this value to check+ * whether current block is packed format or VInt. When packed format, payloads and+ * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when+ * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).+ * <li>SingletonDocID is an optimization when a term only appears in one document. In this+ * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a+ * VIntBlock at that location, the single document ID is written to the term dictionary.+ * </ul>+ * </dl>+ *+ * <a id="Termindex"></a>+ *+ * <dl>+ * <dd><b>Term Index</b>+ * <p>The .tip file contains an index into the term dictionary, so that it can be accessed+ * randomly. See {@link Lucene103BlockTreeTermsWriter} for more details on the format.+ * </dl>+ *+ * <a id="Frequencies"></a>+ *+ * <dl>+ * <dd><b>Frequencies and Skip Data</b>+ * <p>The .doc file contains the lists of documents which contain each term, along with the+ * frequency of the term in that document (except when frequencies are omitted: {@link+ * IndexOptions#DOCS}). Skip data is saved at the end of each term's postings. The skip data+ * is saved once for the entire postings list.+ * <ul>+ * <li>docFile(.doc) --> Header, <TermFreqs><sup>TermCount</sup>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>TermFreqs --> <PackedBlock32> <sup>PackedDocBlockNum/32</sup>, VIntBlock?+ * <li>PackedBlock32 --> Level1SkipData, <PackedBlock> <sup>32</sup>+ * <li>PackedBlock --> Level0SkipData, PackedDocDeltaBlock, PackedFreqBlock?+ * <li>VIntBlock -->+ * <DocDelta[,Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>+ * <li>Level1SkipData --> DocDelta, DocFPDelta, Skip1NumBytes?, ImpactLength?, Impacts?,+ * PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?+ * <li>Level0SkipData --> Skip0NumBytes, DocDelta, DocFPDelta, PackedBlockLength,+ * ImpactLength?, Impacts?, PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?+ * <li>PackedFreqBlock --> {@link PackedInts PackedInts}, uses patching+ * <li>PackedDocDeltaBlock --> {@link PackedInts PackedInts}, does not use patching+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>PackedDocDeltaBlock is theoretically generated from two steps:+ * <ol>+ * <li>Calculate the difference between each document number and previous one, and get+ * a d-gaps list (for the first document, use absolute value);+ * <li>For those d-gaps from first one to+ * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed+ * blocks.+ * </ol>+ * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.+ * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a+ * format that encodes DocDelta and Freq:+ * <p>DocDelta: if frequencies are indexed, this determines both the document number and+ * the frequency. In particular, DocDelta/2 is the difference between this document+ * number and the previous document number (or zero when this is the first document in a+ * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the+ * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the+ * gap (not multiplied by 2) between document numbers and no frequency information is+ * stored.+ * <p>For example, the TermFreqs for a term which occurs once in document seven and+ * three times in document eleven, with frequencies indexed, would be the following+ * sequence of VInts:+ * <p>15, 8, 3+ * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence+ * of VInts instead:+ * <p>7,4+ * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or+ * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)+ * <li>On skip data, DocDelta is the delta between the last doc of the previous block - or+ * -1 if there is no previous block - and the last doc of this block. This helps know by+ * how much the doc ID should be incremented in case the block gets skipped.+ * <li>Skip0Length is the length of skip data at level 0. Encoding it is useful when skip+ * data is never needed to quickly skip over skip data, e.g. if only using nextDoc(). It+ * is also used when only the first fields of skip data are needed, in order to skip+ * over remaining fields without reading them.+ * <li>ImpactLength and Impacts are only stored if frequencies are indexed.+ * <li>Since positions and payloads are also block encoded, the skip should skip to related+ * block first, then fetch the values according to in-block offset. PosFPSkip and+ * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.+ * While PosBlockOffset indicates which value to fetch inside the related block+ * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as+ * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,+ * and stored as a difference sequence.+ * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to+ * the sum of the payload lengths in the current block up to PosBlockOffset+ * <li>ImpactLength is the total length of CompetitiveFreqDelta and CompetitiveNormDelta+ * pairs. CompetitiveFreqDelta and CompetitiveNormDelta are used to safely skip score+ * calculation for uncompetitive documents; See {@link+ * org.apache.lucene.codecs.CompetitiveImpactAccumulator} for more details.+ * </ul>+ * </dl>+ *+ * <a id="Positions"></a>+ *+ * <dl>+ * <dd><b>Positions</b>+ * <p>The .pos file contains the lists of positions that each term occurs at within documents.+ * It also sometimes stores part of payloads and offsets for speedup.+ * <ul>+ * <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>TermPositions --> <PackedPosDeltaBlock> <sup>PackedPosBlockNum</sup>,+ * VIntBlock?+ * <li>VIntBlock --> <PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,+ * OffsetLength?><sup>PosVIntCount</sup>+ * <li>PackedPosDeltaBlock --> {@link PackedInts PackedInts}+ * <li>PositionDelta, OffsetDelta, OffsetLength --> {@link DataOutput#writeVInt VInt}+ * <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayLength</sup>+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and+ * position values for each term document pair are incremental, and ordered by document+ * number.+ * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,+ * payloads or offsets. In particular, PackedPosBlockNum =+ * floor(totalTermFreq/PackedBlockSize)+ * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,+ * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize+ * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock+ * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.+ * <li>PositionDelta is, if payloads are disabled for the term's field, the difference+ * between the position of the current occurrence in the document and the previous+ * occurrence (or zero, if this is the first occurrence in this document). If payloads+ * are enabled for the term's field, then PositionDelta/2 is the difference between the+ * current and the previous position. If payloads are enabled and PositionDelta is odd,+ * then PayloadLength is stored, indicating the length of the payload at the current+ * term position.+ * <li>For example, the TermPositions for a term which occurs as the fourth term in one+ * document, and as the fifth and ninth term in a subsequent document, would be the+ * following sequence of VInts (payloads disabled):+ * <p>4, 5, 4+ * <li>PayloadData is metadata associated with the current term position. If PayloadLength+ * is stored at the current position, then it indicates the length of this payload. If+ * PayloadLength is not stored, then this payload has the same length as the payload at+ * the previous position.+ * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous+ * occurrence (or zero, if this is the first occurrence in this document). If+ * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous+ * occurrence and an OffsetLength follows. Offset data is only written for {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.+ * </ul>+ * </dl>+ *+ * <a id="Payloads"></a>+ *+ * <dl>+ * <dd><b>Payloads and Offsets</b>+ * <p>The .pay file will store payloads and offsets associated with certain term-document+ * positions. Some payloads and offsets will be separated out into .pos file, for performance+ * reasons.+ * <ul>+ * <li>PayFile(.pay): --> Header, <TermPayloads?, TermOffsets?>+ * <sup>TermCount</sup>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>TermPayloads --> <PackedPayLengthBlock, SumPayLength, PayData>+ * <sup>PackedPayBlockNum</sup>+ * <li>TermOffsets --> <PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock>+ * <sup>PackedPayBlockNum</sup>+ * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock -->+ * {@link PackedInts PackedInts}+ * <li>SumPayLength --> {@link DataOutput#writeVInt VInt}+ * <li>PayData --> {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that+ * part of payload/offsets are stored in .pos.+ * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is+ * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip+ * Data</a>. While PackedStartDeltaBlock follows a same procedure as+ * PackedDocDeltaBlock.+ * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also+ * synonym for PackedOffsetBlockNum.+ * <li>SumPayLength is the total length of payloads written within one block, should be the+ * sum of PayLengths in one packed block.+ * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the+ * current position.+ * </ul>+ * </dl>+ *+ * @lucene.experimental+ */+public final class Lucene104PostingsFormat extends PostingsFormat {++ /** Filename extension for some small metadata about how postings are encoded. */+ public static final String META_EXTENSION = "psm";++ /**+ * Filename extension for document number, frequencies, and skip data. See chapter: <a+ * href="#Frequencies">Frequencies and Skip Data</a>+ */+ public static final String DOC_EXTENSION = "doc";++ /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */+ public static final String POS_EXTENSION = "pos";++ /**+ * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and+ * Offsets</a>+ */+ public static final String PAY_EXTENSION = "pay";++ /** Size of blocks. */+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;++ static final int BLOCK_MASK = BLOCK_SIZE - 1;++ /** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */+ public static final int LEVEL1_FACTOR = 32;++ /** Total number of docs covered by level 1 skip data: 32 * 256 = 8,192 */+ public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;++ static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;++ /**+ * Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is+ * internally used to help the JVM make good inlining decisions.+ *+ * @lucene.internal+ */+ public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {+ return Lucene104PostingsReader.BlockPostingsEnum.class;+ }++ static final String TERMS_CODEC = "Lucene104PostingsWriterTerms";+ static final String META_CODEC = "Lucene104PostingsWriterMeta";+ static final String DOC_CODEC = "Lucene104PostingsWriterDoc";+ static final String POS_CODEC = "Lucene104PostingsWriterPos";+ static final String PAY_CODEC = "Lucene104PostingsWriterPay";++ static final int VERSION_START = 0;++ static final int VERSION_CURRENT = VERSION_START;++ private final int version;+ private final int minTermBlockSize;+ private final int maxTermBlockSize;++ /** Creates {@code Lucene104PostingsFormat} with default settings. */+ public Lucene104PostingsFormat() {+ this(+ Lucene103BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,+ Lucene103BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);+ }++ /**+ * Creates {@code Lucene104PostingsFormat} with custom values for {@code minBlockSize} and {@code+ * maxBlockSize} passed to block terms dictionary.+ *+ * @see+ * Lucene103BlockTreeTermsWriter#Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)+ */+ public Lucene104PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {+ this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);+ }++ /** Expert constructor that allows setting the version. */+ public Lucene104PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {+ super("Lucene104");+ if (version < VERSION_START || version > VERSION_CURRENT) {+ throw new IllegalArgumentException("Version out of range: " + version);+ }+ this.version = version;+ Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);+ this.minTermBlockSize = minTermBlockSize;+ this.maxTermBlockSize = maxTermBlockSize;+ }++ @Override+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state, version);+ try {+ return new Lucene103BlockTreeTermsWriter(+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, postingsWriter);+ throw t;+ }+ }++ @Override+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);+ try {+ return new Lucene103BlockTreeTermsReader(postingsReader, state);+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, postingsReader);+ throw t;+ }+ }++ /**+ * Holds all state required for {@link Lucene104PostingsReader} to produce a {@link+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.+ *+ * @lucene.internal+ */+ public static final class IntBlockTermState extends BlockTermState {+ /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */+ public long docStartFP;++ /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */+ public long posStartFP;++ /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */+ public long payStartFP;++ /**+ * file offset for the last position in the last block, if there are more than {@link+ * ForUtil#BLOCK_SIZE} positions; otherwise -1+ *+ * <p>One might think to use total term frequency to track how many positions are left to read+ * as we decode the blocks, and decode the last block differently when num_left_positions <+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip+ * blocks as the skipper will only tell us new position offset (start of block) and number of+ * positions to skip for that block, without telling us how many positions it has skipped.+ */+ public long lastPosBlockOffset;++ /**+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly+ * totalTermFreq in this case.+ */+ public int singletonDocID;++ /** Sole constructor. */+ public IntBlockTermState() {+ lastPosBlockOffset = -1;+ singletonDocID = -1;+ }++ @Override+ public IntBlockTermState clone() {+ IntBlockTermState other = new IntBlockTermState();+ other.copyFrom(this);+ return other;+ }++ @Override+ public void copyFrom(TermState _other) {+ super.copyFrom(_other);+ IntBlockTermState other = (IntBlockTermState) _other;+ docStartFP = other.docStartFP;+ posStartFP = other.posStartFP;+ payStartFP = other.payStartFP;+ lastPosBlockOffset = other.lastPosBlockOffset;+ singletonDocID = other.singletonDocID;+ }++ @Override+ public String toString() {+ return super.toString()+ + " docStartFP="+ + docStartFP+ + " posStartFP="+ + posStartFP+ + " payStartFP="+ + payStartFP+ + " lastPosBlockOffset="+ + lastPosBlockOffset+ + " singletonDocID="+ + singletonDocID;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsReader.javanew file mode 100644index 00000000000..97d75989a70--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsReader.java@@ -0,0 +1,1498 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import static org.apache.lucene.codecs.lucene104.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.DOC_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.LEVEL1_NUM_DOCS;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.META_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.PAY_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.POS_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.VERSION_CURRENT;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.VERSION_START;++import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.PostingsReaderBase;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FreqAndNormBuffer;+import org.apache.lucene.index.Impacts;+import org.apache.lucene.index.ImpactsEnum;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;+import org.apache.lucene.internal.vectorization.VectorizationProvider;+import org.apache.lucene.search.DocAndFloatFeatureBuffer;+import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BitUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.FixedBitSet;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.VectorUtil;++/**+ * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.+ *+ * @lucene.experimental+ */+public final class Lucene104PostingsReader extends PostingsReaderBase {++ static final VectorizationProvider VECTORIZATION_PROVIDER = VectorizationProvider.getInstance();++ private final IndexInput docIn;+ private final IndexInput posIn;+ private final IndexInput payIn;++ private final int maxNumImpactsAtLevel0;+ private final int maxImpactNumBytesAtLevel0;+ private final int maxNumImpactsAtLevel1;+ private final int maxImpactNumBytesAtLevel1;++ /** Sole constructor. */+ public Lucene104PostingsReader(SegmentReadState state) throws IOException {+ String metaName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.META_EXTENSION);+ final long expectedDocFileLength, expectedPosFileLength, expectedPayFileLength;+ int version;+ try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaName)) {+ try {+ version =+ CodecUtil.checkIndexHeader(+ metaIn,+ META_CODEC,+ VERSION_START,+ VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ maxNumImpactsAtLevel0 = metaIn.readInt();+ maxImpactNumBytesAtLevel0 = metaIn.readInt();+ maxNumImpactsAtLevel1 = metaIn.readInt();+ maxImpactNumBytesAtLevel1 = metaIn.readInt();+ expectedDocFileLength = metaIn.readLong();+ if (state.fieldInfos.hasProx()) {+ expectedPosFileLength = metaIn.readLong();+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {+ expectedPayFileLength = metaIn.readLong();+ } else {+ expectedPayFileLength = -1;+ }+ } else {+ expectedPosFileLength = -1;+ expectedPayFileLength = -1;+ }+ CodecUtil.checkFooter(metaIn, null);+ } catch (Throwable t) {+ if (metaIn != null) {+ CodecUtil.checkFooter(metaIn, t);+ throw new AssertionError("unreachable");+ } else {+ throw t;+ }+ }+ }++ IndexInput docIn = null;+ IndexInput posIn = null;+ IndexInput payIn = null;++ // NOTE: these data files are too costly to verify checksum against all the bytes on open,+ // but for now we at least verify proper structure of the checksum footer: which looks+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption+ // such as file truncation.++ String docName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.DOC_EXTENSION);+ try {+ docIn =+ state.directory.openInput(+ docName, state.context.withHints(FileTypeHint.DATA, FileDataHint.POSTINGS));+ CodecUtil.checkIndexHeader(+ docIn, DOC_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);+ CodecUtil.retrieveChecksum(docIn, expectedDocFileLength);++ if (state.fieldInfos.hasProx()) {+ String proxName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.POS_EXTENSION);+ posIn = state.directory.openInput(proxName, state.context.withHints(FileTypeHint.DATA));+ CodecUtil.checkIndexHeader(+ posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);+ CodecUtil.retrieveChecksum(posIn, expectedPosFileLength);++ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {+ String payName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name,+ state.segmentSuffix,+ Lucene104PostingsFormat.PAY_EXTENSION);+ payIn = state.directory.openInput(payName, state.context.withHints(FileTypeHint.DATA));+ CodecUtil.checkIndexHeader(+ payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);+ CodecUtil.retrieveChecksum(payIn, expectedPayFileLength);+ }+ }++ this.docIn = docIn;+ this.posIn = posIn;+ this.payIn = payIn;+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, docIn, posIn, payIn);+ throw t;+ }+ }++ @Override+ public void init(IndexInput termsIn, SegmentReadState state) throws IOException {+ // Make sure we are talking to the matching postings writer+ CodecUtil.checkIndexHeader(+ termsIn,+ TERMS_CODEC,+ VERSION_START,+ VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ final int indexBlockSize = termsIn.readVInt();+ if (indexBlockSize != BLOCK_SIZE) {+ throw new IllegalStateException(+ "index-time BLOCK_SIZE ("+ + indexBlockSize+ + ") != read-time BLOCK_SIZE ("+ + BLOCK_SIZE+ + ")");+ }+ }++ static void prefixSum(int[] buffer, int count, long base) {+ buffer[0] += base;+ for (int i = 1; i < count; ++i) {+ buffer[i] += buffer[i - 1];+ }+ }++ @Override+ public BlockTermState newTermState() {+ return new IntBlockTermState();+ }++ @Override+ public void close() throws IOException {+ IOUtils.close(docIn, posIn, payIn);+ }++ @Override+ public void decodeTerm(+ DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)+ throws IOException {+ final IntBlockTermState termState = (IntBlockTermState) _termState;+ if (absolute) {+ termState.docStartFP = 0;+ termState.posStartFP = 0;+ termState.payStartFP = 0;+ }++ final long l = in.readVLong();+ if ((l & 0x01) == 0) {+ termState.docStartFP += l >>> 1;+ if (termState.docFreq == 1) {+ termState.singletonDocID = in.readVInt();+ } else {+ termState.singletonDocID = -1;+ }+ } else {+ assert absolute == false;+ assert termState.singletonDocID != -1;+ termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);+ }++ if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {+ termState.posStartFP += in.readVLong();+ if (fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0+ || fieldInfo.hasPayloads()) {+ termState.payStartFP += in.readVLong();+ }+ if (termState.totalTermFreq > BLOCK_SIZE) {+ termState.lastPosBlockOffset = in.readVLong();+ } else {+ termState.lastPosBlockOffset = -1;+ }+ }+ }++ @Override+ public PostingsEnum postings(+ FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)+ throws IOException {+ return (reuse instanceof BlockPostingsEnum everythingEnum+ && everythingEnum.canReuse(docIn, fieldInfo, flags, false)+ ? everythingEnum+ : new BlockPostingsEnum(fieldInfo, flags, false))+ .reset((IntBlockTermState) termState, flags);+ }++ @Override+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)+ throws IOException {+ return new BlockPostingsEnum(fieldInfo, flags, true).reset((IntBlockTermState) state, flags);+ }++ private static int sumOverRange(int[] arr, int start, int end) {+ int res = 0;+ for (int i = start; i < end; i++) {+ res += arr[i];+ }+ return res;+ }++ final class BlockPostingsEnum extends ImpactsEnum {++ private enum DeltaEncoding {+ /**+ * Deltas between consecutive docs are stored as packed integers, ie. the block is encoded+ * using Frame Of Reference (FOR).+ */+ PACKED,+ /**+ * Deltas between consecutive docs are stored using unary coding, ie. {@code delta-1} zero+ * bits followed by a one bit, ie. the block is encoded as an offset plus a bit set.+ */+ UNARY+ }++ private ForUtil forUtil;+ private PForUtil pforUtil;++ /* Variables that store the content of a block and the current position within this block */+ /* Shared variables */+ private DeltaEncoding encoding;+ private int doc; // doc we last read++ /* Variables when the block is stored as packed deltas (Frame Of Reference) */+ private final int[] docBuffer = new int[BLOCK_SIZE];++ /* Variables when the block is stored as a bit set */+ // Since we use a bit set when it's more storage-efficient, the bit set cannot have more than+ // BLOCK_SIZE*32 bits, which is the maximum possible storage requirement with FOR.+ private final FixedBitSet docBitSet = new FixedBitSet(BLOCK_SIZE * Integer.SIZE);+ private int docBitSetBase;+ // Reuse docBuffer for cumulative pop counts of the words of the bit set.+ private final int[] docCumulativeWordPopCounts = docBuffer;++ // level 0 skip data+ private int level0LastDocID;+ private long level0DocEndFP;++ // level 1 skip data+ private int level1LastDocID;+ private long level1DocEndFP;+ private int level1DocCountUpto;++ private int docFreq; // number of docs in this posting list+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)++ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1++ private int docCountLeft; // number of remaining docs in this postings list+ private int prevDocID; // last doc ID of the previous block++ private int docBufferSize;+ private int docBufferUpto;++ private IndexInput docIn;+ private PostingDecodingUtil docInUtil;++ private final int[] freqBuffer = new int[BLOCK_SIZE];+ private final int[] posDeltaBuffer;++ private final int[] payloadLengthBuffer;+ private final int[] offsetStartDeltaBuffer;+ private final int[] offsetLengthBuffer;++ private byte[] payloadBytes;+ private int payloadByteUpto;+ private int payloadLength;++ private int lastStartOffset;+ private int startOffset;+ private int endOffset;++ private int posBufferUpto;++ final IndexInput posIn;+ final PostingDecodingUtil posInUtil;+ final IndexInput payIn;+ final PostingDecodingUtil payInUtil;+ final BytesRef payload;++ final IndexOptions options;+ final boolean indexHasFreq;+ final boolean indexHasPos;+ final boolean indexHasOffsets;+ final boolean indexHasPayloads;+ final boolean indexHasOffsetsOrPayloads;++ final int flags;+ final boolean needsFreq;+ final boolean needsPos;+ final boolean needsOffsets;+ final boolean needsPayloads;+ final boolean needsOffsetsOrPayloads;+ final boolean needsImpacts;+ final boolean needsDocsAndFreqsOnly;++ private long freqFP; // offset of the freq block++ private int position; // current position++ // value of docBufferUpto on the last doc ID when positions have been read+ private int posDocBufferUpto;++ // how many positions "behind" we are; nextPosition must+ // skip these to "catch up":+ private int posPendingCount;++ // File pointer where the last (vInt encoded) pos delta+ // block is. We need this to know whether to bulk+ // decode vs vInt decode the block:+ private long lastPosBlockFP;++ // level 0 skip data+ private long level0PosEndFP;+ private int level0BlockPosUpto;+ private long level0PayEndFP;+ private int level0BlockPayUpto;+ private final BytesRef level0SerializedImpacts;++ // level 1 skip data+ private long level1PosEndFP;+ private int level1BlockPosUpto;+ private long level1PayEndFP;+ private int level1BlockPayUpto;+ private final BytesRef level1SerializedImpacts;++ private final FreqAndNormBuffer impactBuffer;++ // true if we shallow-advanced to a new block that we have not decoded yet+ private boolean needsRefilling;++ public BlockPostingsEnum(FieldInfo fieldInfo, int flags, boolean needsImpacts)+ throws IOException {+ options = fieldInfo.getIndexOptions();+ indexHasFreq = options.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;+ indexHasPos = options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;+ indexHasOffsets =+ options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;+ indexHasPayloads = fieldInfo.hasPayloads();+ indexHasOffsetsOrPayloads = indexHasOffsets || indexHasPayloads;++ this.flags = flags;+ needsFreq = indexHasFreq && PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);+ needsPos = indexHasPos && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);+ needsOffsets = indexHasOffsets && PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);+ needsPayloads =+ indexHasPayloads && PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);+ needsOffsetsOrPayloads = needsOffsets || needsPayloads;+ this.needsImpacts = needsImpacts;+ needsDocsAndFreqsOnly = needsPos == false && needsImpacts == false;++ if (needsFreq == false) {+ Arrays.fill(freqBuffer, 1);+ }++ if (needsImpacts) {+ impactBuffer = new FreqAndNormBuffer();+ int capacity = 1; // for dummy impacts+ if (needsFreq) {+ capacity = Math.max(maxNumImpactsAtLevel0, capacity);+ capacity = Math.max(maxNumImpactsAtLevel1, capacity);+ }+ impactBuffer.growNoCopy(capacity);+ } else {+ impactBuffer = null;+ }++ if (needsFreq && needsImpacts) {+ level0SerializedImpacts = new BytesRef(maxImpactNumBytesAtLevel0);+ level1SerializedImpacts = new BytesRef(maxImpactNumBytesAtLevel1);+ } else {+ level0SerializedImpacts = null;+ level1SerializedImpacts = null;+ }++ if (needsPos) {+ this.posIn = Lucene104PostingsReader.this.posIn.clone();+ posInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(posIn);+ posDeltaBuffer = new int[BLOCK_SIZE];+ } else {+ this.posIn = null;+ this.posInUtil = null;+ posDeltaBuffer = null;+ }++ if (needsOffsets || needsPayloads) {+ this.payIn = Lucene104PostingsReader.this.payIn.clone();+ payInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(payIn);+ } else {+ this.payIn = null;+ payInUtil = null;+ }++ if (needsOffsets) {+ offsetStartDeltaBuffer = new int[BLOCK_SIZE];+ offsetLengthBuffer = new int[BLOCK_SIZE];+ } else {+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ startOffset = -1;+ endOffset = -1;+ }++ if (indexHasPayloads) {+ payloadLengthBuffer = new int[BLOCK_SIZE];+ payloadBytes = new byte[128];+ payload = new BytesRef();+ } else {+ payloadLengthBuffer = null;+ payloadBytes = null;+ payload = null;+ }+ }++ public boolean canReuse(+ IndexInput docIn, FieldInfo fieldInfo, int flags, boolean needsImpacts) {+ return docIn == Lucene104PostingsReader.this.docIn+ && options == fieldInfo.getIndexOptions()+ && indexHasPayloads == fieldInfo.hasPayloads()+ && this.flags == flags+ && this.needsImpacts == needsImpacts;+ }++ public BlockPostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {+ docFreq = termState.docFreq;+ singletonDocID = termState.singletonDocID;+ if (docFreq > 1) {+ if (docIn == null) {+ // lazy init+ docIn = Lucene104PostingsReader.this.docIn.clone();+ docInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(docIn);+ }+ prefetchPostings(docIn, termState);+ }++ if (forUtil == null && docFreq >= BLOCK_SIZE) {+ forUtil = new ForUtil();+ }+ totalTermFreq = indexHasFreq ? termState.totalTermFreq : termState.docFreq;+ if (needsFreq && pforUtil == null && totalTermFreq >= BLOCK_SIZE) {+ if (forUtil == null) {+ forUtil = new ForUtil();+ }+ pforUtil = new PForUtil(forUtil);+ }++ // Where this term's postings start in the .pos file:+ final long posTermStartFP = termState.posStartFP;+ // Where this term's payloads/offsets start in the .pay+ // file:+ final long payTermStartFP = termState.payStartFP;+ if (posIn != null) {+ posIn.seek(posTermStartFP);+ if (payIn != null) {+ payIn.seek(payTermStartFP);+ }+ }+ level1PosEndFP = posTermStartFP;+ level1PayEndFP = payTermStartFP;+ level0PosEndFP = posTermStartFP;+ level0PayEndFP = payTermStartFP;+ posPendingCount = 0;+ payloadByteUpto = 0;+ if (termState.totalTermFreq < BLOCK_SIZE) {+ lastPosBlockFP = posTermStartFP;+ } else if (termState.totalTermFreq == BLOCK_SIZE) {+ lastPosBlockFP = -1;+ } else {+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;+ }++ level1BlockPosUpto = 0;+ level1BlockPayUpto = 0;+ level0BlockPosUpto = 0;+ level0BlockPayUpto = 0;+ posBufferUpto = BLOCK_SIZE;++ doc = -1;+ prevDocID = -1;+ docCountLeft = docFreq;+ freqFP = -1L;+ level0LastDocID = -1;+ if (docFreq < LEVEL1_NUM_DOCS) {+ level1LastDocID = NO_MORE_DOCS;+ if (docFreq > 1) {+ docIn.seek(termState.docStartFP);+ }+ } else {+ level1LastDocID = -1;+ level1DocEndFP = termState.docStartFP;+ }+ level1DocCountUpto = 0;+ docBufferSize = BLOCK_SIZE;+ docBufferUpto = BLOCK_SIZE;+ posDocBufferUpto = BLOCK_SIZE;++ return this;+ }++ @Override+ public int docID() {+ return doc;+ }++ @Override+ public int freq() throws IOException {+ if (freqFP != -1) {+ docIn.seek(freqFP);+ pforUtil.decode(docInUtil, freqBuffer);+ freqFP = -1;+ }+ return freqBuffer[docBufferUpto - 1];+ }++ private void refillFullBlock() throws IOException {+ int bitsPerValue = docIn.readByte();+ if (bitsPerValue > 0) {+ // block is encoded as 256 packed integers that record the delta between doc IDs+ forUtil.decode(bitsPerValue, docInUtil, docBuffer);+ int sum = prevDocID;+ for (int i = 0; i < BLOCK_SIZE; ++i) {+ sum += docBuffer[i];+ docBuffer[i] = sum;+ }+ encoding = DeltaEncoding.PACKED;+ } else {+ // block is encoded as a bit set+ assert level0LastDocID != NO_MORE_DOCS;+ docBitSetBase = prevDocID + 1;+ int numLongs;+ if (bitsPerValue == 0) {+ // 0 is used to record that all 256 docs in the block are consecutive+ numLongs = BLOCK_SIZE / Long.SIZE; // 2+ docBitSet.set(0, BLOCK_SIZE);+ } else {+ numLongs = -bitsPerValue;+ docIn.readLongs(docBitSet.getBits(), 0, numLongs);+ }+ if (needsFreq) {+ // Note: we know that BLOCK_SIZE bits are set, so no need to compute the cumulative pop+ // count at the last index, it will be BLOCK_SIZE.+ // Note: this for loop auto-vectorizes+ for (int i = 0; i < numLongs - 1; ++i) {+ docCumulativeWordPopCounts[i] = Long.bitCount(docBitSet.getBits()[i]);+ }+ for (int i = 1; i < numLongs - 1; ++i) {+ docCumulativeWordPopCounts[i] += docCumulativeWordPopCounts[i - 1];+ }+ docCumulativeWordPopCounts[numLongs - 1] = BLOCK_SIZE;+ assert docCumulativeWordPopCounts[numLongs - 2]+ + Long.bitCount(docBitSet.getBits()[numLongs - 1])+ == BLOCK_SIZE;+ }+ encoding = DeltaEncoding.UNARY;+ }+ if (indexHasFreq) {+ if (needsFreq) {+ freqFP = docIn.getFilePointer();+ }+ PForUtil.skip(docIn);+ }+ docCountLeft -= BLOCK_SIZE;+ prevDocID = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ posDocBufferUpto = 0;+ }++ private void refillRemainder() throws IOException {+ assert docCountLeft >= 0 && docCountLeft < BLOCK_SIZE;+ if (docFreq == 1) {+ docBuffer[0] = singletonDocID;+ freqBuffer[0] = (int) totalTermFreq;+ docBuffer[1] = NO_MORE_DOCS;+ assert freqFP == -1;+ docCountLeft = 0;+ docBufferSize = 1;+ } else {+ // Read vInts:+ PostingsUtil.readVIntBlock(+ docIn, docBuffer, freqBuffer, docCountLeft, indexHasFreq, needsFreq);+ prefixSum(docBuffer, docCountLeft, prevDocID);+ docBuffer[docCountLeft] = NO_MORE_DOCS;+ freqFP = -1L;+ docBufferSize = docCountLeft;+ docCountLeft = 0;+ }+ prevDocID = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ posDocBufferUpto = 0;+ encoding = DeltaEncoding.PACKED;+ assert docBuffer[docBufferSize] == NO_MORE_DOCS;+ }++ private void refillDocs() throws IOException {+ assert docCountLeft >= 0;++ if (docCountLeft >= BLOCK_SIZE) {+ refillFullBlock();+ } else {+ refillRemainder();+ }+ }++ private void skipLevel1To(int target) throws IOException {+ while (true) {+ prevDocID = level1LastDocID;+ level0LastDocID = level1LastDocID;+ docIn.seek(level1DocEndFP);+ level0PosEndFP = level1PosEndFP;+ level0BlockPosUpto = level1BlockPosUpto;+ level0PayEndFP = level1PayEndFP;+ level0BlockPayUpto = level1BlockPayUpto;+ docCountLeft = docFreq - level1DocCountUpto;+ level1DocCountUpto += LEVEL1_NUM_DOCS;++ if (docCountLeft < LEVEL1_NUM_DOCS) {+ level1LastDocID = NO_MORE_DOCS;+ break;+ }++ level1LastDocID += docIn.readVInt();+ long delta = docIn.readVLong();+ level1DocEndFP = delta + docIn.getFilePointer();++ if (indexHasFreq) {+ long skip1EndFP = docIn.readShort() + docIn.getFilePointer();+ int numImpactBytes = docIn.readShort();+ if (needsImpacts && level1LastDocID >= target) {+ docIn.readBytes(level1SerializedImpacts.bytes, 0, numImpactBytes);+ level1SerializedImpacts.length = numImpactBytes;+ } else {+ docIn.skipBytes(numImpactBytes);+ }+ if (indexHasPos) {+ level1PosEndFP += docIn.readVLong();+ level1BlockPosUpto = docIn.readByte() & 0xFF;+ if (indexHasOffsetsOrPayloads) {+ level1PayEndFP += docIn.readVLong();+ level1BlockPayUpto = docIn.readVInt();+ }+ }+ assert docIn.getFilePointer() == skip1EndFP;+ }++ if (level1LastDocID >= target) {+ break;+ }+ }+ }++ private void doMoveToNextLevel0Block() throws IOException {+ assert doc == level0LastDocID;+ if (posIn != null) {+ if (level0PosEndFP >= posIn.getFilePointer()) {+ posIn.seek(level0PosEndFP);+ posPendingCount = level0BlockPosUpto;+ if (payIn != null) {+ assert level0PayEndFP >= payIn.getFilePointer();+ payIn.seek(level0PayEndFP);+ payloadByteUpto = level0BlockPayUpto;+ }+ posBufferUpto = BLOCK_SIZE;+ } else {+ assert freqFP == -1L;+ posPendingCount += sumOverRange(freqBuffer, posDocBufferUpto, BLOCK_SIZE);+ }+ }++ if (docCountLeft >= BLOCK_SIZE) {+ docIn.readVLong(); // level0NumBytes+ int docDelta = readVInt15(docIn);+ level0LastDocID += docDelta;+ long blockLength = readVLong15(docIn);+ level0DocEndFP = docIn.getFilePointer() + blockLength;+ if (indexHasFreq) {+ int numImpactBytes = docIn.readVInt();+ if (needsImpacts) {+ docIn.readBytes(level0SerializedImpacts.bytes, 0, numImpactBytes);+ level0SerializedImpacts.length = numImpactBytes;+ } else {+ docIn.skipBytes(numImpactBytes);+ }++ if (indexHasPos) {+ level0PosEndFP += docIn.readVLong();+ level0BlockPosUpto = docIn.readByte() & 0xFF;+ if (indexHasOffsetsOrPayloads) {+ level0PayEndFP += docIn.readVLong();+ level0BlockPayUpto = docIn.readVInt();+ }+ }+ }+ refillFullBlock();+ } else {+ level0LastDocID = NO_MORE_DOCS;+ refillRemainder();+ }+ }++ private void moveToNextLevel0Block() throws IOException {+ if (doc == level1LastDocID) { // advance level 1 skip data+ skipLevel1To(doc + 1);+ }++ // Now advance level 0 skip data+ prevDocID = level0LastDocID;++ if (needsDocsAndFreqsOnly && docCountLeft >= BLOCK_SIZE) {+ // Optimize the common path for exhaustive evaluation+ long level0NumBytes = docIn.readVLong();+ long level0End = docIn.getFilePointer() + level0NumBytes;+ level0LastDocID += readVInt15(docIn);+ docIn.seek(level0End);+ refillFullBlock();+ } else {+ doMoveToNextLevel0Block();+ }+ }++ private void readLevel0PosData() throws IOException {+ level0PosEndFP += docIn.readVLong();+ level0BlockPosUpto = docIn.readByte() & 0xFF;+ if (indexHasOffsetsOrPayloads) {+ level0PayEndFP += docIn.readVLong();+ level0BlockPayUpto = docIn.readVInt();+ }+ }++ private void seekPosData(long posFP, int posUpto, long payFP, int payUpto) throws IOException {+ // If nextBlockPosFP is less than the current FP, it means that the block of positions for+ // the first docs of the next block are already decoded. In this case we just accumulate+ // frequencies into posPendingCount instead of seeking backwards and decoding the same pos+ // block again.+ if (posFP >= posIn.getFilePointer()) {+ posIn.seek(posFP);+ posPendingCount = posUpto;+ if (payIn != null) { // needs payloads or offsets+ assert level0PayEndFP >= payIn.getFilePointer();+ payIn.seek(payFP);+ payloadByteUpto = payUpto;+ }+ posBufferUpto = BLOCK_SIZE;+ } else {+ posPendingCount += sumOverRange(freqBuffer, posDocBufferUpto, BLOCK_SIZE);+ }+ }++ private void skipLevel0To(int target) throws IOException {+ long posFP;+ int posUpto;+ long payFP;+ int payUpto;++ while (true) {+ prevDocID = level0LastDocID;++ posFP = level0PosEndFP;+ posUpto = level0BlockPosUpto;+ payFP = level0PayEndFP;+ payUpto = level0BlockPayUpto;++ if (docCountLeft >= BLOCK_SIZE) {+ long numSkipBytes = docIn.readVLong();+ long skip0End = docIn.getFilePointer() + numSkipBytes;+ int docDelta = readVInt15(docIn);+ level0LastDocID += docDelta;+ boolean found = target <= level0LastDocID;+ long blockLength = readVLong15(docIn);+ level0DocEndFP = docIn.getFilePointer() + blockLength;++ if (indexHasFreq) {+ if (found == false && needsPos == false) {+ docIn.seek(skip0End);+ } else {+ int numImpactBytes = docIn.readVInt();+ if (needsImpacts && found) {+ docIn.readBytes(level0SerializedImpacts.bytes, 0, numImpactBytes);+ level0SerializedImpacts.length = numImpactBytes;+ } else {+ docIn.skipBytes(numImpactBytes);+ }++ if (needsPos) {+ readLevel0PosData();+ } else {+ docIn.seek(skip0End);+ }+ }+ }++ if (found) {+ break;+ }++ docIn.seek(level0DocEndFP);+ docCountLeft -= BLOCK_SIZE;+ } else {+ level0LastDocID = NO_MORE_DOCS;+ break;+ }+ }++ if (posIn != null) { // needs positions+ seekPosData(posFP, posUpto, payFP, payUpto);+ }+ }++ @Override+ public void advanceShallow(int target) throws IOException {+ if (target > level0LastDocID) { // advance level 0 skip data+ doAdvanceShallow(target);+ needsRefilling = true;+ }+ }++ private void doAdvanceShallow(int target) throws IOException {+ if (target > level1LastDocID) { // advance skip data on level 1+ skipLevel1To(target);+ } else if (needsRefilling) {+ docIn.seek(level0DocEndFP);+ docCountLeft -= BLOCK_SIZE;+ }++ skipLevel0To(target);+ }++ @Override+ public int nextDoc() throws IOException {+ if (doc == level0LastDocID || needsRefilling) {+ if (needsRefilling) {+ refillDocs();+ needsRefilling = false;+ } else {+ moveToNextLevel0Block();+ }+ }++ switch (encoding) {+ case PACKED:+ doc = docBuffer[docBufferUpto];+ break;+ case UNARY:+ int next = docBitSet.nextSetBit(doc - docBitSetBase + 1);+ assert next != NO_MORE_DOCS;+ doc = docBitSetBase + next;+ break;+ }++ ++docBufferUpto;+ return this.doc;+ }++ @Override+ public int advance(int target) throws IOException {+ if (target > level0LastDocID || needsRefilling) {+ if (target > level0LastDocID) {+ doAdvanceShallow(target);+ }+ refillDocs();+ needsRefilling = false;+ }++ switch (encoding) {+ case PACKED:+ {+ int next = VectorUtil.findNextGEQ(docBuffer, target, docBufferUpto, docBufferSize);+ this.doc = docBuffer[next];+ docBufferUpto = next + 1;+ }+ break;+ case UNARY:+ {+ int next = docBitSet.nextSetBit(target - docBitSetBase);+ assert next != NO_MORE_DOCS;+ this.doc = docBitSetBase + next;+ if (needsFreq) {+ int wordIndex = next >> 6;+ // Take the cumulative pop count for the given word, and subtract bits on the left of+ // the current doc.+ docBufferUpto =+ 1+ + docCumulativeWordPopCounts[wordIndex]+ - Long.bitCount(docBitSet.getBits()[wordIndex] >>> next);+ } else {+ // When only docs needed and block is UNARY encoded, we do not need to maintain+ // docBufferUpTo to record the iteration position in the block.+ // docBufferUpTo == 0 means the block has not been iterated.+ // docBufferUpTo != 0 means the block has been iterated.+ docBufferUpto = 1;+ }+ }+ break;+ }++ return doc;+ }++ @Override+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {+ if (doc >= upTo) {+ return;+ }++ // Handle the current doc separately, it may be on the previous docBuffer.+ bitSet.set(doc - offset);++ for (; ; ) {+ if (doc == level0LastDocID) {+ // refill+ moveToNextLevel0Block();+ }++ switch (encoding) {+ case PACKED:+ {+ int start = docBufferUpto;+ int end = computeBufferEndBoundary(upTo);+ if (end != 0) {+ bufferIntoBitSet(start, end, bitSet, offset);+ doc = docBuffer[end - 1];+ }+ docBufferUpto = end;+ if (end != BLOCK_SIZE) {+ // Either the block is a tail block, or the block did not fully match, we're done.+ nextDoc();+ assert doc >= upTo;+ return;+ }+ }+ break;+ case UNARY:+ {+ int sourceFrom;+ if (docBufferUpto == 0) {+ // start from beginning+ sourceFrom = 0;+ } else {+ // start after the current doc+ sourceFrom = doc - docBitSetBase + 1;+ }++ int destFrom = docBitSetBase - offset + sourceFrom;++ assert level0LastDocID != NO_MORE_DOCS;+ int sourceTo = Math.min(upTo, level0LastDocID + 1) - docBitSetBase;++ if (sourceTo > sourceFrom) {+ FixedBitSet.orRange(docBitSet, sourceFrom, bitSet, destFrom, sourceTo - sourceFrom);+ }+ if (docBitSetBase + sourceTo <= level0LastDocID) {+ // We stopped before the end of the current bit set, which means that we're done.+ // Set the current doc before returning.+ advance(docBitSetBase + sourceTo);+ return;+ }+ doc = level0LastDocID;+ docBufferUpto = BLOCK_SIZE;+ }+ break;+ }+ }+ }++ @Override+ public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {+ assert needsRefilling == false;++ if (needsFreq == false) {+ super.nextPostings(upTo, buffer);+ return;+ }++ buffer.size = 0;+ if (doc >= upTo) {+ return;+ }++ // Only return docs from the current block+ // +1 to make BitSetUtil#denseBitsetToArray work, see its java doc.+ buffer.growNoCopy(BLOCK_SIZE + 1);+ upTo = (int) Math.min(upTo, level0LastDocID + 1L);++ // Frequencies are decoded lazily, calling freq() makes sure that the freq block is decoded+ freq();++ int start = docBufferUpto - 1;+ switch (encoding) {+ case PACKED:+ int end = computeBufferEndBoundary(upTo);+ buffer.size = end - start;+ System.arraycopy(docBuffer, start, buffer.docs, 0, buffer.size);+ break;+ case UNARY:+ buffer.size =+ BitSetUtil.denseBitsetToArray(+ docBitSet, doc - docBitSetBase, upTo - docBitSetBase, docBitSetBase, buffer.docs);+ break;+ }++ assert buffer.size > 0;+ for (int i = 0; i < buffer.size; ++i) {+ buffer.features[i] = freqBuffer[start + i];+ }++ advance(upTo);+ }++ private int computeBufferEndBoundary(int upTo) {+ if (docBufferSize != 0 && docBuffer[docBufferSize - 1] < upTo) {+ // All docs in the buffer are under upTo+ return docBufferSize;+ } else {+ // Find the index of the first doc that is greater than or equal to upTo+ return VectorUtil.findNextGEQ(docBuffer, upTo, docBufferUpto, docBufferSize);+ }+ }++ private void bufferIntoBitSet(int start, int end, FixedBitSet bitSet, int offset)+ throws IOException {+ // bitSet#set and `doc - offset` get auto-vectorized+ for (int i = start; i < end; ++i) {+ int doc = docBuffer[i];+ bitSet.set(doc - offset);+ }+ }++ @Override+ public int docIDRunEnd() throws IOException {+ if (encoding == DeltaEncoding.UNARY) {+ // Note: this assumes that BLOCK_SIZE == 256, this bit of the code would need to be changed+ // if+ // the block size was changed.+ // Hack to avoid compiler warning that both sides of the equal sign are identical.+ long blockSize = BLOCK_SIZE;+ assert blockSize == 4 * Long.SIZE;+ boolean level0IsDense = true;+ for (int i = 0; i < 4; ++i) {+ if (docBitSet.getBits()[i] != -1L) {+ level0IsDense = false;+ break;+ }+ }+ if (level0IsDense) {++ int level0DocCountUpto = docFreq - docCountLeft;+ boolean level1IsDense =+ level1LastDocID - level0LastDocID == level1DocCountUpto - level0DocCountUpto;+ if (level1IsDense) {+ return level1LastDocID + 1;+ }++ return level0LastDocID + 1;+ }+ }++ return super.docIDRunEnd();+ }++ private void skipPositions(int freq) throws IOException {+ // Skip positions now:+ int toSkip = posPendingCount - freq;+ // if (DEBUG) {+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);+ // }++ final int leftInBlock = BLOCK_SIZE - posBufferUpto;+ if (toSkip < leftInBlock) {+ int end = posBufferUpto + toSkip;+ if (needsPayloads) {+ payloadByteUpto += sumOverRange(payloadLengthBuffer, posBufferUpto, end);+ }+ posBufferUpto = end;+ } else {+ toSkip -= leftInBlock;+ while (toSkip >= BLOCK_SIZE) {+ assert posIn.getFilePointer() != lastPosBlockFP;+ PForUtil.skip(posIn);++ if (payIn != null) {+ if (indexHasPayloads) {+ // Skip payloadLength block:+ PForUtil.skip(payIn);++ // Skip payloadBytes block:+ int numBytes = payIn.readVInt();+ payIn.seek(payIn.getFilePointer() + numBytes);+ }++ if (indexHasOffsets) {+ PForUtil.skip(payIn);+ PForUtil.skip(payIn);+ }+ }+ toSkip -= BLOCK_SIZE;+ }+ refillPositions();+ if (needsPayloads) {+ payloadByteUpto = sumOverRange(payloadLengthBuffer, 0, toSkip);+ }+ posBufferUpto = toSkip;+ }+ }++ private void refillLastPositionBlock() throws IOException {+ final int count = (int) (totalTermFreq % BLOCK_SIZE);+ int payloadLength = 0;+ int offsetLength = 0;+ payloadByteUpto = 0;+ for (int i = 0; i < count; i++) {+ int code = posIn.readVInt();+ if (indexHasPayloads) {+ if ((code & 1) != 0) {+ payloadLength = posIn.readVInt();+ }+ if (payloadLengthBuffer != null) { // needs payloads+ payloadLengthBuffer[i] = payloadLength;+ posDeltaBuffer[i] = code >>> 1;+ if (payloadLength != 0) {+ if (payloadByteUpto + payloadLength > payloadBytes.length) {+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);+ }+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);+ payloadByteUpto += payloadLength;+ }+ } else {+ posIn.skipBytes(payloadLength);+ }+ } else {+ posDeltaBuffer[i] = code;+ }++ if (indexHasOffsets) {+ int deltaCode = posIn.readVInt();+ if ((deltaCode & 1) != 0) {+ offsetLength = posIn.readVInt();+ }+ if (offsetStartDeltaBuffer != null) { // needs offsets+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;+ offsetLengthBuffer[i] = offsetLength;+ }+ }+ }+ payloadByteUpto = 0;+ }++ private void refillOffsetsOrPayloads() throws IOException {+ if (indexHasPayloads) {+ if (needsPayloads) {+ pforUtil.decode(payInUtil, payloadLengthBuffer);+ int numBytes = payIn.readVInt();++ if (numBytes > payloadBytes.length) {+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);+ }+ payIn.readBytes(payloadBytes, 0, numBytes);+ } else if (payIn != null) { // needs offsets+ // this works, because when writing a vint block we always force the first length to be+ // written+ PForUtil.skip(payIn); // skip over lengths+ int numBytes = payIn.readVInt(); // read length of payloadBytes+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes+ }+ payloadByteUpto = 0;+ }++ if (indexHasOffsets) {+ if (needsOffsets) {+ pforUtil.decode(payInUtil, offsetStartDeltaBuffer);+ pforUtil.decode(payInUtil, offsetLengthBuffer);+ } else if (payIn != null) { // needs payloads+ // this works, because when writing a vint block we always force the first length to be+ // written+ PForUtil.skip(payIn); // skip over starts+ PForUtil.skip(payIn); // skip over lengths+ }+ }+ }++ private void refillPositions() throws IOException {+ if (posIn.getFilePointer() == lastPosBlockFP) {+ refillLastPositionBlock();+ return;+ }+ pforUtil.decode(posInUtil, posDeltaBuffer);++ if (indexHasOffsetsOrPayloads) {+ refillOffsetsOrPayloads();+ }+ }++ private void accumulatePendingPositions() throws IOException {+ int freq = freq(); // trigger lazy decoding of freqs+ posPendingCount += sumOverRange(freqBuffer, posDocBufferUpto, docBufferUpto);+ posDocBufferUpto = docBufferUpto;++ assert posPendingCount > 0;++ if (posPendingCount > freq) {+ skipPositions(freq);+ posPendingCount = freq;+ }+ }++ private void accumulatePayloadAndOffsets() {+ if (needsPayloads) {+ payloadLength = payloadLengthBuffer[posBufferUpto];+ payload.bytes = payloadBytes;+ payload.offset = payloadByteUpto;+ payload.length = payloadLength;+ payloadByteUpto += payloadLength;+ }++ if (needsOffsets) {+ startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];+ endOffset = startOffset + offsetLengthBuffer[posBufferUpto];+ lastStartOffset = startOffset;+ }+ }++ @Override+ public int nextPosition() throws IOException {+ if (needsPos == false) {+ return -1;+ }++ assert posDocBufferUpto <= docBufferUpto;+ if (posDocBufferUpto != docBufferUpto) {+ // First position we're reading on this doc+ accumulatePendingPositions();+ position = 0;+ lastStartOffset = 0;+ }++ if (posBufferUpto == BLOCK_SIZE) {+ refillPositions();+ posBufferUpto = 0;+ }+ position += posDeltaBuffer[posBufferUpto];++ if (needsOffsetsOrPayloads) {+ accumulatePayloadAndOffsets();+ }++ posBufferUpto++;+ posPendingCount--;+ return position;+ }++ @Override+ public int startOffset() {+ if (needsOffsets == false) {+ return -1;+ }+ return startOffset;+ }++ @Override+ public int endOffset() {+ if (needsOffsets == false) {+ return -1;+ }+ return endOffset;+ }++ @Override+ public BytesRef getPayload() {+ if (needsPayloads == false || payloadLength == 0) {+ return null;+ } else {+ return payload;+ }+ }++ @Override+ public long cost() {+ return docFreq;+ }++ private final Impacts impacts =+ new Impacts() {++ private final ByteArrayDataInput scratch = new ByteArrayDataInput();++ @Override+ public int numLevels() {+ return indexHasFreq == false || level1LastDocID == NO_MORE_DOCS ? 1 : 2;+ }++ @Override+ public int getDocIdUpTo(int level) {+ if (indexHasFreq == false) {+ return NO_MORE_DOCS;+ }+ if (level == 0) {+ return level0LastDocID;+ }+ return level == 1 ? level1LastDocID : NO_MORE_DOCS;+ }++ @Override+ public FreqAndNormBuffer getImpacts(int level) {+ if (indexHasFreq == false) {+ // Max freq is 1 since freqs are not indexed+ impactBuffer.size = 1;+ impactBuffer.freqs[0] = 1;+ impactBuffer.norms[0] = 1L;+ return impactBuffer;+ }+ if (level == 0 && level0LastDocID != NO_MORE_DOCS) {+ return readImpacts(level0SerializedImpacts, impactBuffer);+ }+ if (level == 1) {+ return readImpacts(level1SerializedImpacts, impactBuffer);+ }+ impactBuffer.size = 1;+ impactBuffer.freqs[0] = Integer.MAX_VALUE;+ impactBuffer.norms[0] = 1L;+ return impactBuffer;+ }++ private FreqAndNormBuffer readImpacts(+ BytesRef serialized, FreqAndNormBuffer impactBuffer) {+ var scratch = this.scratch;+ scratch.reset(serialized.bytes, 0, serialized.length);+ Lucene104PostingsReader.readImpacts(scratch, impactBuffer);+ return impactBuffer;+ }+ };++ @Override+ public Impacts getImpacts() {+ assert needsImpacts;+ return impacts;+ }+ }++ /**+ * @see Lucene104PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)+ */+ static int readVInt15(DataInput in) throws IOException {+ short s = in.readShort();+ if (s >= 0) {+ return s;+ } else {+ return (s & 0x7FFF) | (in.readVInt() << 15);+ }+ }++ /**+ * @see Lucene104PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)+ */+ static long readVLong15(DataInput in) throws IOException {+ short s = in.readShort();+ if (s >= 0) {+ return s;+ } else {+ return (s & 0x7FFFL) | (in.readVLong() << 15);+ }+ }++ private static void prefetchPostings(IndexInput docIn, IntBlockTermState state)+ throws IOException {+ assert state.docFreq > 1; // Singletons are inlined in the terms dict, nothing to prefetch+ if (docIn.getFilePointer() != state.docStartFP) {+ // Don't prefetch if the input is already positioned at the right offset, which suggests that+ // the caller is streaming the entire inverted index (e.g. for merging), let the read-ahead+ // logic do its work instead. Note that this heuristic doesn't work for terms that have skip+ // data, since skip data is stored after the last term, but handling all terms that have <256+ // docs is a good start already.+ docIn.prefetch(state.docStartFP, 1);+ }+ // Note: we don't prefetch positions or offsets, which are less likely to be needed.+ }++ static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer reuse) {+ int freq = 0;+ long norm = 0;+ int size = 0;+ while (in.getPosition() < in.length()) {+ int freqDelta = in.readVInt();+ if ((freqDelta & 0x01) != 0) {+ freq += 1 + (freqDelta >>> 1);+ try {+ norm += 1 + in.readZLong();+ } catch (IOException e) {+ throw new RuntimeException(e); // cannot happen on a BADI+ }+ } else {+ freq += 1 + (freqDelta >>> 1);+ norm++;+ }+ reuse.freqs[size] = freq;+ reuse.norms[size] = norm;+ size++;+ }+ reuse.size = size;+ return reuse;+ }++ @Override+ public void checkIntegrity() throws IOException {+ if (docIn != null) {+ CodecUtil.checksumEntireFile(docIn);+ }+ if (posIn != null) {+ CodecUtil.checksumEntireFile(posIn);+ }+ if (payIn != null) {+ CodecUtil.checksumEntireFile(payIn);+ }+ }++ @Override+ public String toString() {+ return getClass().getSimpleName()+ + "(positions="+ + (posIn != null)+ + ",payloads="+ + (payIn != null)+ + ")";+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsWriter.javanew file mode 100644index 00000000000..ddae0e62755--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsWriter.java@@ -0,0 +1,734 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.DOC_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.LEVEL1_MASK;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.META_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.PAY_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.POS_CODEC;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.TERMS_CODEC;++import java.io.IOException;+import java.util.Arrays;+import java.util.Collection;+import java.util.List;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;+import org.apache.lucene.codecs.Impact;+import org.apache.lucene.codecs.PushPostingsWriterBase;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.store.ByteBuffersDataOutput;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BitUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.FixedBitSet;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.packed.PackedInts;++/** Writer for {@link Lucene104PostingsFormat}. */+public class Lucene104PostingsWriter extends PushPostingsWriterBase {++ static final IntBlockTermState EMPTY_STATE = new IntBlockTermState();++ private final int version;++ IndexOutput metaOut;+ IndexOutput docOut;+ IndexOutput posOut;+ IndexOutput payOut;++ IntBlockTermState lastState;++ // Holds starting file pointers for current term:+ private long docStartFP;+ private long posStartFP;+ private long payStartFP;++ final int[] docDeltaBuffer;+ final int[] freqBuffer;+ private int docBufferUpto;++ final int[] posDeltaBuffer;+ final int[] payloadLengthBuffer;+ final int[] offsetStartDeltaBuffer;+ final int[] offsetLengthBuffer;+ private int posBufferUpto;++ private byte[] payloadBytes;+ private int payloadByteUpto;++ private int level0LastDocID;+ private long level0LastPosFP;+ private long level0LastPayFP;++ private int level1LastDocID;+ private long level1LastPosFP;+ private long level1LastPayFP;++ private int docID;+ private int lastDocID;+ private int lastPosition;+ private int lastStartOffset;+ private int docCount;++ private final PForUtil pforUtil;+ private final ForUtil forUtil;++ private boolean fieldHasNorms;+ private NumericDocValues norms;+ private final CompetitiveImpactAccumulator level0FreqNormAccumulator =+ new CompetitiveImpactAccumulator();+ private final CompetitiveImpactAccumulator level1CompetitiveFreqNormAccumulator =+ new CompetitiveImpactAccumulator();++ private int maxNumImpactsAtLevel0;+ private int maxImpactNumBytesAtLevel0;+ private int maxNumImpactsAtLevel1;+ private int maxImpactNumBytesAtLevel1;++ /** Scratch output that we use to be able to prepend the encoded length, e.g. impacts. */+ private final ByteBuffersDataOutput scratchOutput = ByteBuffersDataOutput.newResettableInstance();++ /**+ * Output for a single block. This is useful to be able to prepend skip data before each block,+ * which can only be computed once the block is encoded. The content is then typically copied to+ * {@link #level1Output}.+ */+ private final ByteBuffersDataOutput level0Output = ByteBuffersDataOutput.newResettableInstance();++ /**+ * Output for groups of 32 blocks. This is useful to prepend skip data for these 32 blocks, which+ * can only be done once we have encoded these 32 blocks. The content is then typically copied to+ * {@link #docCount}.+ */+ private final ByteBuffersDataOutput level1Output = ByteBuffersDataOutput.newResettableInstance();++ /**+ * Reusable FixedBitSet, for dense blocks that are more efficiently stored by storing them as a+ * bit set than as packed deltas.+ */+ // Since we use a bit set when it's more storage-efficient, the bit set cannot have more than+ // BLOCK_SIZE*32 bits, which is the maximum possible storage requirement with FOR.+ private final FixedBitSet spareBitSet = new FixedBitSet(BLOCK_SIZE * Integer.SIZE);++ /** Sole public constructor. */+ public Lucene104PostingsWriter(SegmentWriteState state) throws IOException {+ this(state, Lucene104PostingsFormat.VERSION_CURRENT);+ }++ /** Constructor that takes a version. */+ Lucene104PostingsWriter(SegmentWriteState state, int version) throws IOException {+ this.version = version;+ String metaFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.META_EXTENSION);+ String docFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.DOC_EXTENSION);+ metaOut = state.directory.createOutput(metaFileName, state.context);+ IndexOutput posOut = null;+ IndexOutput payOut = null;+ try {+ docOut = state.directory.createOutput(docFileName, state.context);+ CodecUtil.writeIndexHeader(+ metaOut, META_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);+ CodecUtil.writeIndexHeader(+ docOut, DOC_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);+ forUtil = new ForUtil();+ pforUtil = new PForUtil(forUtil);+ if (state.fieldInfos.hasProx()) {+ posDeltaBuffer = new int[BLOCK_SIZE];+ String posFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.POS_EXTENSION);+ posOut = state.directory.createOutput(posFileName, state.context);+ CodecUtil.writeIndexHeader(+ posOut, POS_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);++ if (state.fieldInfos.hasPayloads()) {+ payloadBytes = new byte[128];+ payloadLengthBuffer = new int[BLOCK_SIZE];+ } else {+ payloadBytes = null;+ payloadLengthBuffer = null;+ }++ if (state.fieldInfos.hasOffsets()) {+ offsetStartDeltaBuffer = new int[BLOCK_SIZE];+ offsetLengthBuffer = new int[BLOCK_SIZE];+ } else {+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ }++ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {+ String payFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name,+ state.segmentSuffix,+ Lucene104PostingsFormat.PAY_EXTENSION);+ payOut = state.directory.createOutput(payFileName, state.context);+ CodecUtil.writeIndexHeader(+ payOut, PAY_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);+ }+ } else {+ posDeltaBuffer = null;+ payloadLengthBuffer = null;+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ payloadBytes = null;+ }+ this.payOut = payOut;+ this.posOut = posOut;+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, metaOut, docOut, posOut, payOut);+ throw t;+ }++ docDeltaBuffer = new int[BLOCK_SIZE];+ freqBuffer = new int[BLOCK_SIZE];+ }++ @Override+ public IntBlockTermState newTermState() {+ return new IntBlockTermState();+ }++ @Override+ public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {+ CodecUtil.writeIndexHeader(+ termsOut, TERMS_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);+ termsOut.writeVInt(BLOCK_SIZE);+ }++ @Override+ public void setField(FieldInfo fieldInfo) {+ super.setField(fieldInfo);+ lastState = EMPTY_STATE;+ fieldHasNorms = fieldInfo.hasNorms();+ }++ @Override+ public void startTerm(NumericDocValues norms) {+ docStartFP = docOut.getFilePointer();+ if (writePositions) {+ posStartFP = posOut.getFilePointer();+ level1LastPosFP = level0LastPosFP = posStartFP;+ if (writePayloads || writeOffsets) {+ payStartFP = payOut.getFilePointer();+ level1LastPayFP = level0LastPayFP = payStartFP;+ }+ }+ lastDocID = -1;+ level0LastDocID = -1;+ level1LastDocID = -1;+ this.norms = norms;+ if (writeFreqs) {+ level0FreqNormAccumulator.clear();+ }+ }++ @Override+ public void startDoc(int docID, int termDocFreq) throws IOException {+ if (docBufferUpto == BLOCK_SIZE) {+ flushDocBlock(false);+ docBufferUpto = 0;+ }++ final int docDelta = docID - lastDocID;++ if (docID < 0 || docDelta <= 0) {+ throw new CorruptIndexException(+ "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);+ }++ docDeltaBuffer[docBufferUpto] = docDelta;+ if (writeFreqs) {+ freqBuffer[docBufferUpto] = termDocFreq;+ }++ this.docID = docID;+ lastPosition = 0;+ lastStartOffset = 0;++ if (writeFreqs) {+ long norm;+ if (fieldHasNorms) {+ boolean found = norms.advanceExact(docID);+ if (found == false) {+ // This can happen if indexing hits a problem after adding a doc to the+ // postings but before buffering the norm. Such documents are written+ // deleted and will go away on the first merge.+ norm = 1L;+ } else {+ norm = norms.longValue();+ assert norm != 0 : docID;+ }+ } else {+ norm = 1L;+ }++ level0FreqNormAccumulator.add(termDocFreq, norm);+ }+ }++ @Override+ public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)+ throws IOException {+ if (position > IndexWriter.MAX_POSITION) {+ throw new CorruptIndexException(+ "position="+ + position+ + " is too large (> IndexWriter.MAX_POSITION="+ + IndexWriter.MAX_POSITION+ + ")",+ docOut);+ }+ if (position < 0) {+ throw new CorruptIndexException("position=" + position + " is < 0", docOut);+ }+ posDeltaBuffer[posBufferUpto] = position - lastPosition;+ if (writePayloads) {+ if (payload == null || payload.length == 0) {+ // no payload+ payloadLengthBuffer[posBufferUpto] = 0;+ } else {+ payloadLengthBuffer[posBufferUpto] = payload.length;+ if (payloadByteUpto + payload.length > payloadBytes.length) {+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);+ }+ System.arraycopy(+ payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);+ payloadByteUpto += payload.length;+ }+ }++ if (writeOffsets) {+ assert startOffset >= lastStartOffset;+ assert endOffset >= startOffset;+ offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;+ offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;+ lastStartOffset = startOffset;+ }++ posBufferUpto++;+ lastPosition = position;+ if (posBufferUpto == BLOCK_SIZE) {+ pforUtil.encode(posDeltaBuffer, posOut);++ if (writePayloads) {+ pforUtil.encode(payloadLengthBuffer, payOut);+ payOut.writeVInt(payloadByteUpto);+ payOut.writeBytes(payloadBytes, 0, payloadByteUpto);+ payloadByteUpto = 0;+ }+ if (writeOffsets) {+ pforUtil.encode(offsetStartDeltaBuffer, payOut);+ pforUtil.encode(offsetLengthBuffer, payOut);+ }+ posBufferUpto = 0;+ }+ }++ @Override+ public void finishDoc() {+ docBufferUpto++;+ docCount++;++ lastDocID = docID;+ }++ /**+ * Special vints that are encoded on 2 bytes if they require 15 bits or less. VInt becomes+ * especially slow when the number of bytes is variable, so this special layout helps in the case+ * when the number likely requires 15 bits or less+ */+ static void writeVInt15(DataOutput out, int v) throws IOException {+ assert v >= 0;+ writeVLong15(out, v);+ }++ /**+ * @see #writeVInt15(DataOutput, int)+ */+ static void writeVLong15(DataOutput out, long v) throws IOException {+ assert v >= 0;+ if ((v & ~0x7FFFL) == 0) {+ out.writeShort((short) v);+ } else {+ out.writeShort((short) (0x8000 | (v & 0x7FFF)));+ out.writeVLong(v >> 15);+ }+ }++ private void flushDocBlock(boolean finishTerm) throws IOException {+ assert docBufferUpto != 0;++ if (docBufferUpto < BLOCK_SIZE) {+ assert finishTerm;+ PostingsUtil.writeVIntBlock(+ level0Output, docDeltaBuffer, freqBuffer, docBufferUpto, writeFreqs);+ } else {+ if (writeFreqs) {+ List<Impact> impacts = level0FreqNormAccumulator.getCompetitiveFreqNormPairs();+ if (impacts.size() > maxNumImpactsAtLevel0) {+ maxNumImpactsAtLevel0 = impacts.size();+ }+ writeImpacts(impacts, scratchOutput);+ assert level0Output.size() == 0;+ if (scratchOutput.size() > maxImpactNumBytesAtLevel0) {+ maxImpactNumBytesAtLevel0 = Math.toIntExact(scratchOutput.size());+ }+ level0Output.writeVLong(scratchOutput.size());+ scratchOutput.copyTo(level0Output);+ scratchOutput.reset();+ if (writePositions) {+ level0Output.writeVLong(posOut.getFilePointer() - level0LastPosFP);+ level0Output.writeByte((byte) posBufferUpto);+ level0LastPosFP = posOut.getFilePointer();++ if (writeOffsets || writePayloads) {+ level0Output.writeVLong(payOut.getFilePointer() - level0LastPayFP);+ level0Output.writeVInt(payloadByteUpto);+ level0LastPayFP = payOut.getFilePointer();+ }+ }+ }+ long numSkipBytes = level0Output.size();+ // Now we need to decide whether to encode block deltas as packed integers (FOR) or unary+ // codes (bit set). FOR makes #nextDoc() a bit faster while the bit set approach makes+ // #advance() usually faster and #intoBitSet() much faster. In the end, we make the decision+ // based on storage requirements, picking the bit set approach whenever it's more+ // storage-efficient than the next number of bits per value (which effectively slightly biases+ // towards the bit set approach).+ int or = 0;+ for (int i : docDeltaBuffer) {+ or |= i;+ }+ assert or != 0;+ int bitsPerValue = PackedInts.bitsRequired(or);+ int docRange = lastDocID - level0LastDocID;+ assert docRange == Arrays.stream(docDeltaBuffer).sum();+ int numBitSetLongs = FixedBitSet.bits2words(docRange);+ int numBitsNextBitsPerValue = Math.min(Integer.SIZE, bitsPerValue + 1) * BLOCK_SIZE;+ if (docRange == BLOCK_SIZE) {+ level0Output.writeByte((byte) 0);+ } else if (numBitsNextBitsPerValue <= docRange) {+ level0Output.writeByte((byte) bitsPerValue);+ forUtil.encode(docDeltaBuffer, bitsPerValue, level0Output);+ } else {+ // Storing doc deltas is more efficient using unary coding (ie. storing doc IDs as a bit+ // set)+ spareBitSet.clear(0, numBitSetLongs << 6);+ int s = -1;+ for (int i : docDeltaBuffer) {+ s += i;+ spareBitSet.set(s);+ }+ // We never use the bit set encoding when it requires more than Integer.SIZE=32 bits per+ // value. So the bit set cannot have more than BLOCK_SIZE * Integer.SIZE / Long.SIZE = 64+ // longs, which fits on a byte.+ assert numBitSetLongs <= BLOCK_SIZE / 2;+ level0Output.writeByte((byte) -numBitSetLongs);+ for (int i = 0; i < numBitSetLongs; ++i) {+ level0Output.writeLong(spareBitSet.getBits()[i]);+ }+ }++ if (writeFreqs) {+ pforUtil.encode(freqBuffer, level0Output);+ }++ // docID - lastBlockDocID is at least 256, so it can never fit a single byte with a vint+ // Even if we subtracted 256, only extremely dense blocks would be eligible to a single byte+ // so let's go with 2 bytes right away+ writeVInt15(scratchOutput, docID - level0LastDocID);+ writeVLong15(scratchOutput, level0Output.size());+ numSkipBytes += scratchOutput.size();+ level1Output.writeVLong(numSkipBytes);+ scratchOutput.copyTo(level1Output);+ scratchOutput.reset();+ }++ level0Output.copyTo(level1Output);+ level0Output.reset();+ level0LastDocID = docID;+ if (writeFreqs) {+ level1CompetitiveFreqNormAccumulator.addAll(level0FreqNormAccumulator);+ level0FreqNormAccumulator.clear();+ }++ if ((docCount & LEVEL1_MASK) == 0) { // true every 32 blocks (4,096 docs)+ writeLevel1SkipData();+ level1LastDocID = docID;+ level1CompetitiveFreqNormAccumulator.clear();+ } else if (finishTerm) {+ level1Output.copyTo(docOut);+ level1Output.reset();+ level1CompetitiveFreqNormAccumulator.clear();+ }+ }++ private void writeLevel1SkipData() throws IOException {+ docOut.writeVInt(docID - level1LastDocID);+ final long level1End;+ if (writeFreqs) {+ List<Impact> impacts = level1CompetitiveFreqNormAccumulator.getCompetitiveFreqNormPairs();+ if (impacts.size() > maxNumImpactsAtLevel1) {+ maxNumImpactsAtLevel1 = impacts.size();+ }+ writeImpacts(impacts, scratchOutput);+ long numImpactBytes = scratchOutput.size();+ if (numImpactBytes > maxImpactNumBytesAtLevel1) {+ maxImpactNumBytesAtLevel1 = Math.toIntExact(numImpactBytes);+ }+ if (writePositions) {+ scratchOutput.writeVLong(posOut.getFilePointer() - level1LastPosFP);+ scratchOutput.writeByte((byte) posBufferUpto);+ level1LastPosFP = posOut.getFilePointer();+ if (writeOffsets || writePayloads) {+ scratchOutput.writeVLong(payOut.getFilePointer() - level1LastPayFP);+ scratchOutput.writeVInt(payloadByteUpto);+ level1LastPayFP = payOut.getFilePointer();+ }+ }+ final long level1Len = 2 * Short.BYTES + scratchOutput.size() + level1Output.size();+ docOut.writeVLong(level1Len);+ level1End = docOut.getFilePointer() + level1Len;+ // There are at most 256 impacts, that require at most 2 bytes each+ assert numImpactBytes <= Short.MAX_VALUE;+ // Like impacts plus a few vlongs, still way under the max short value+ assert scratchOutput.size() + Short.BYTES <= Short.MAX_VALUE;+ docOut.writeShort((short) (scratchOutput.size() + Short.BYTES));+ docOut.writeShort((short) numImpactBytes);+ scratchOutput.copyTo(docOut);+ scratchOutput.reset();+ } else {+ docOut.writeVLong(level1Output.size());+ level1End = docOut.getFilePointer() + level1Output.size();+ }+ level1Output.copyTo(docOut);+ level1Output.reset();+ assert docOut.getFilePointer() == level1End : docOut.getFilePointer() + " " + level1End;+ }++ static void writeImpacts(Collection<Impact> impacts, DataOutput out) throws IOException {+ Impact previous = new Impact(0, 0);+ for (Impact impact : impacts) {+ assert impact.freq > previous.freq;+ assert Long.compareUnsigned(impact.norm, previous.norm) > 0;+ int freqDelta = impact.freq - previous.freq - 1;+ long normDelta = impact.norm - previous.norm - 1;+ if (normDelta == 0) {+ // most of time, norm only increases by 1, so we can fold everything in a single byte+ out.writeVInt(freqDelta << 1);+ } else {+ out.writeVInt((freqDelta << 1) | 1);+ out.writeZLong(normDelta);+ }+ previous = impact;+ }+ }++ /** Called when we are done adding docs to this term */+ @Override+ public void finishTerm(BlockTermState _state) throws IOException {+ IntBlockTermState state = (IntBlockTermState) _state;+ assert state.docFreq > 0;++ // TODO: wasteful we are counting this (counting # docs+ // for this term) in two places?+ assert state.docFreq == docCount : state.docFreq + " vs " + docCount;++ // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to+ // it.+ final int singletonDocID;+ if (state.docFreq == 1) {+ // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq+ singletonDocID = docDeltaBuffer[0] - 1;+ } else {+ singletonDocID = -1;+ flushDocBlock(true);+ }++ final long lastPosBlockOffset;++ if (writePositions) {+ // totalTermFreq is just total number of positions(or payloads, or offsets)+ // associated with current term.+ assert state.totalTermFreq != -1;+ if (state.totalTermFreq > BLOCK_SIZE) {+ // record file offset for last pos in last block+ lastPosBlockOffset = posOut.getFilePointer() - posStartFP;+ } else {+ lastPosBlockOffset = -1;+ }+ if (posBufferUpto > 0) {+ assert posBufferUpto < BLOCK_SIZE;+ // TODO: should we send offsets/payloads to+ // .pay...? seems wasteful (have to store extra+ // vLong for low (< BLOCK_SIZE) DF terms = vast vast+ // majority)++ // vInt encode the remaining positions/payloads/offsets:+ int lastPayloadLength = -1; // force first payload length to be written+ int lastOffsetLength = -1; // force first offset length to be written+ int payloadBytesReadUpto = 0;+ for (int i = 0; i < posBufferUpto; i++) {+ final int posDelta = posDeltaBuffer[i];+ if (writePayloads) {+ final int payloadLength = payloadLengthBuffer[i];+ if (payloadLength != lastPayloadLength) {+ lastPayloadLength = payloadLength;+ posOut.writeVInt((posDelta << 1) | 1);+ posOut.writeVInt(payloadLength);+ } else {+ posOut.writeVInt(posDelta << 1);+ }++ if (payloadLength != 0) {+ posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);+ payloadBytesReadUpto += payloadLength;+ }+ } else {+ posOut.writeVInt(posDelta);+ }++ if (writeOffsets) {+ int delta = offsetStartDeltaBuffer[i];+ int length = offsetLengthBuffer[i];+ if (length == lastOffsetLength) {+ posOut.writeVInt(delta << 1);+ } else {+ posOut.writeVInt(delta << 1 | 1);+ posOut.writeVInt(length);+ lastOffsetLength = length;+ }+ }+ }++ if (writePayloads) {+ assert payloadBytesReadUpto == payloadByteUpto;+ payloadByteUpto = 0;+ }+ }+ } else {+ lastPosBlockOffset = -1;+ }++ state.docStartFP = docStartFP;+ state.posStartFP = posStartFP;+ state.payStartFP = payStartFP;+ state.singletonDocID = singletonDocID;++ state.lastPosBlockOffset = lastPosBlockOffset;+ docBufferUpto = 0;+ posBufferUpto = 0;+ lastDocID = -1;+ docCount = 0;+ }++ @Override+ public void encodeTerm(+ DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)+ throws IOException {+ IntBlockTermState state = (IntBlockTermState) _state;+ if (absolute) {+ lastState = EMPTY_STATE;+ assert lastState.docStartFP == 0;+ }++ if (lastState.singletonDocID != -1+ && state.singletonDocID != -1+ && state.docStartFP == lastState.docStartFP) {+ // With runs of rare values such as ID fields, the increment of pointers in the docs file is+ // often 0.+ // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we+ // encode the delta+ // between consecutive doc IDs to save space.+ final long delta = (long) state.singletonDocID - lastState.singletonDocID;+ out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);+ } else {+ out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);+ if (state.singletonDocID != -1) {+ out.writeVInt(state.singletonDocID);+ }+ }++ if (writePositions) {+ out.writeVLong(state.posStartFP - lastState.posStartFP);+ if (writePayloads || writeOffsets) {+ out.writeVLong(state.payStartFP - lastState.payStartFP);+ }+ }+ if (writePositions) {+ if (state.lastPosBlockOffset != -1) {+ out.writeVLong(state.lastPosBlockOffset);+ }+ }+ lastState = state;+ }++ @Override+ public void close() throws IOException {+ // TODO: add a finish() at least to PushBase? DV too...?+ try {+ try {+ if (docOut != null) {+ CodecUtil.writeFooter(docOut);+ }+ if (posOut != null) {+ CodecUtil.writeFooter(posOut);+ }+ if (payOut != null) {+ CodecUtil.writeFooter(payOut);+ }+ if (metaOut != null) {+ metaOut.writeInt(maxNumImpactsAtLevel0);+ metaOut.writeInt(maxImpactNumBytesAtLevel0);+ metaOut.writeInt(maxNumImpactsAtLevel1);+ metaOut.writeInt(maxImpactNumBytesAtLevel1);+ metaOut.writeLong(docOut.getFilePointer());+ if (posOut != null) {+ metaOut.writeLong(posOut.getFilePointer());+ if (payOut != null) {+ metaOut.writeLong(payOut.getFilePointer());+ }+ }+ CodecUtil.writeFooter(metaOut);+ }+ } catch (Throwable t) {+ IOUtils.closeWhileSuppressingExceptions(t, metaOut, docOut, posOut, payOut);+ throw t;+ }+ IOUtils.close(metaOut, docOut, posOut, payOut);+ } finally {+ metaOut = docOut = posOut = payOut = null;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PForUtil.javanew file mode 100644index 00000000000..7985040c599--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PForUtil.java@@ -0,0 +1,135 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.packed.PackedInts;++/** Utility class to encode sequences of 256 small positive integers. */+final class PForUtil {++ private static final int MAX_EXCEPTIONS = 7;++ static boolean allEqual(int[] l) {+ for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {+ if (l[i] != l[0]) {+ return false;+ }+ }+ return true;+ }++ static {+ assert ForUtil.BLOCK_SIZE <= 256 : "blocksize must fit in one byte. got " + ForUtil.BLOCK_SIZE;+ }++ private final ForUtil forUtil;++ PForUtil(ForUtil forUtil) {+ this.forUtil = forUtil;+ }++ /** Encode 256 integers from {@code ints} into {@code out}. */+ void encode(int[] ints, DataOutput out) throws IOException {+ // histogram of bit widths+ final int[] histogram = new int[32];+ int maxBitsRequired = 0;+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ final int v = ints[i];+ final int bits = PackedInts.bitsRequired(v);+ histogram[bits]++;+ maxBitsRequired = Math.max(maxBitsRequired, bits);+ }++ // We store patch on a byte, so we can't decrease bits by more than 8+ final int minBits = Math.max(0, maxBitsRequired - 8);+ int cumulativeExceptions = 0;+ int patchedBitsRequired = maxBitsRequired;+ int numExceptions = 0;++ for (int b = maxBitsRequired; b >= minBits; --b) {+ if (cumulativeExceptions > MAX_EXCEPTIONS) {+ break;+ }+ patchedBitsRequired = b;+ numExceptions = cumulativeExceptions;+ cumulativeExceptions += histogram[b];+ }++ final int maxUnpatchedValue = (1 << patchedBitsRequired) - 1;+ final byte[] exceptions = new byte[numExceptions * 2];+ if (numExceptions > 0) {+ int exceptionCount = 0;+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ if (ints[i] > maxUnpatchedValue) {+ exceptions[exceptionCount * 2] = (byte) i;+ exceptions[exceptionCount * 2 + 1] = (byte) (ints[i] >>> patchedBitsRequired);+ ints[i] &= maxUnpatchedValue;+ exceptionCount++;+ }+ }+ assert exceptionCount == numExceptions : exceptionCount + " " + numExceptions;+ }++ if (allEqual(ints) && maxBitsRequired <= 8) {+ for (int i = 0; i < numExceptions; ++i) {+ exceptions[2 * i + 1] =+ (byte) (Byte.toUnsignedInt(exceptions[2 * i + 1]) << patchedBitsRequired);+ }+ out.writeByte((byte) (numExceptions << 5));+ out.writeVInt(ints[0]);+ } else {+ final int token = (numExceptions << 5) | patchedBitsRequired;+ out.writeByte((byte) token);+ forUtil.encode(ints, patchedBitsRequired, out);+ }+ out.writeBytes(exceptions, exceptions.length);+ }++ /** Decode 256 integers into {@code ints}. */+ void decode(PostingDecodingUtil pdu, int[] ints) throws IOException {+ var in = pdu.in;+ final int token = Byte.toUnsignedInt(in.readByte());+ final int bitsPerValue = token & 0x1f;+ if (bitsPerValue == 0) {+ Arrays.fill(ints, 0, ForUtil.BLOCK_SIZE, in.readVInt());+ } else {+ forUtil.decode(bitsPerValue, pdu, ints);+ }+ final int numExceptions = token >>> 5;+ for (int i = 0; i < numExceptions; ++i) {+ ints[Byte.toUnsignedInt(in.readByte())] |= Byte.toUnsignedInt(in.readByte()) << bitsPerValue;+ }+ }++ /** Skip 256 integers. */+ static void skip(DataInput in) throws IOException {+ final int token = Byte.toUnsignedInt(in.readByte());+ final int bitsPerValue = token & 0x1f;+ final int numExceptions = token >>> 5;+ if (bitsPerValue == 0) {+ in.readVLong();+ in.skipBytes((numExceptions << 1));+ } else {+ in.skipBytes(ForUtil.numBytes(bitsPerValue) + (numExceptions << 1));+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingIndexInput.javasimilarity index 74%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.javarename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingIndexInput.javaindex e66ed635146..550f0a17659 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingIndexInput.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;import java.io.IOException;import org.apache.lucene.internal.vectorization.PostingDecodingUtil;@@ -32,26 +32,15 @@ public final class PostingIndexInput {VectorizationProvider.getInstance();public final ForUtil forUtil;- public final ForDeltaUtil forDeltaUtil;private final PostingDecodingUtil postingDecodingUtil;- public PostingIndexInput(IndexInput in, ForUtil forUtil, ForDeltaUtil forDeltaUtil)- throws IOException {+ public PostingIndexInput(IndexInput in, ForUtil forUtil) throws IOException {this.forUtil = forUtil;- this.forDeltaUtil = forDeltaUtil;this.postingDecodingUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);}- /** Decode 128 integers stored on {@code bitsPerValues} bits per value into {@code ints}. */+ /** Decode 256 integers stored on {@code bitsPerValues} bits per value into {@code ints}. */public void decode(int bitsPerValue, int[] ints) throws IOException {forUtil.decode(bitsPerValue, postingDecodingUtil, ints);}-- /**- * Decode 128 integers stored on {@code bitsPerValues} bits per value, compute their prefix sum,- * and store results into {@code ints}.- */- public void decodeAndPrefixSum(int bitsPerValue, int base, int[] ints) throws IOException {- forDeltaUtil.decodeAndPrefixSum(bitsPerValue, postingDecodingUtil, base, ints);- }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingsUtil.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.javarename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingsUtil.javaindex e28d59166dc..ed5f25f060c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingsUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;import java.io.IOException;import org.apache.lucene.store.DataOutput;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.pynew file mode 100644index 00000000000..eb73e4e4e89--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py@@ -0,0 +1,328 @@+#! /usr/bin/env python++# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from math import gcd++"""Code generation for ForUtil.java"""++MAX_SPECIALIZED_BITS_PER_VALUE = 16+OUTPUT_FILE = "ForUtil.java"+PRIMITIVE_SIZE = [8, 16, 32]+HEADER = """// This file has been automatically generated, DO NOT EDIT++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;+import org.apache.lucene.store.DataOutput;++/**+ * Inspired from https://fulmicoton.com/posts/bitpacking/+ * Encodes multiple integers in one to get SIMD-like speedups.+ * If bitsPerValue <= 8 then we pack 4 ints per Java int+ * else if bitsPerValue <= 16 we pack 2 ints per Java int+ * else we do scalar operations.+ */+public final class ForUtil {++ /** Number of integers per block. */+ public static final int BLOCK_SIZE = 256;+ static final int BLOCK_SIZE_LOG2 = 8;++ static int expandMask16(int mask16) {+ return mask16 | (mask16 << 16);+ }++ static int expandMask8(int mask8) {+ return expandMask16(mask8 | (mask8 << 8));+ }++ static int mask32(int bitsPerValue) {+ return (1 << bitsPerValue) - 1;+ }++ static int mask16(int bitsPerValue) {+ return expandMask16((1 << bitsPerValue) - 1);+ }++ static int mask8(int bitsPerValue) {+ return expandMask8((1 << bitsPerValue) - 1);+ }++ static void expand8(int[] arr) {+ for (int i = 0; i < 64; ++i) {+ int l = arr[i];+ arr[i] = (l >>> 24) & 0xFF;+ arr[64 + i] = (l >>> 16) & 0xFF;+ arr[128 + i] = (l >>> 8) & 0xFF;+ arr[192 + i] = l & 0xFF;+ }+ }++ static void collapse8(int[] arr) {+ for (int i = 0; i < 64; ++i) {+ arr[i] =+ (arr[i] << 24)+ | (arr[64 + i] << 16)+ | (arr[128 + i] << 8)+ | arr[192 + i];+ }+ }++ static void expand16(int[] arr) {+ for (int i = 0; i < 128; ++i) {+ int l = arr[i];+ arr[i] = (l >>> 16) & 0xFFFF;+ arr[128 + i] = l & 0xFFFF;+ }+ }++ static void collapse16(int[] arr) {+ for (int i = 0; i < 128; ++i) {+ arr[i] = (arr[i] << 16) | arr[128 + i];+ }+ }++ private final int[] tmp = new int[BLOCK_SIZE];++ /** Encode 256 integers from {@code ints} into {@code out}. */+ void encode(int[] ints, int bitsPerValue, DataOutput out) throws IOException {+ final int nextPrimitive;+ if (bitsPerValue <= 8) {+ nextPrimitive = 8;+ collapse8(ints);+ } else if (bitsPerValue <= 16) {+ nextPrimitive = 16;+ collapse16(ints);+ } else {+ nextPrimitive = 32;+ }+ encode(ints, bitsPerValue, nextPrimitive, out, tmp);+ }++ static void encode(int[] ints, int bitsPerValue, int primitiveSize, DataOutput out, int[] tmp) throws IOException {+ final int numInts = BLOCK_SIZE * primitiveSize / Integer.SIZE;++ final int numIntsPerShift = bitsPerValue * 8;+ int idx = 0;+ int shift = primitiveSize - bitsPerValue;+ for (int i = 0; i < numIntsPerShift; ++i) {+ tmp[i] = ints[idx++] << shift;+ }+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {+ for (int i = 0; i < numIntsPerShift; ++i) {+ tmp[i] |= ints[idx++] << shift;+ }+ }++ final int remainingBitsPerInt = shift + bitsPerValue;+ final int maskRemainingBitsPerInt;+ if (primitiveSize == 8) {+ maskRemainingBitsPerInt = MASKS8[remainingBitsPerInt];+ } else if (primitiveSize == 16) {+ maskRemainingBitsPerInt = MASKS16[remainingBitsPerInt];+ } else {+ maskRemainingBitsPerInt = MASKS32[remainingBitsPerInt];+ }++ int tmpIdx = 0;+ int remainingBitsPerValue = bitsPerValue;+ while (idx < numInts) {+ if (remainingBitsPerValue >= remainingBitsPerInt) {+ remainingBitsPerValue -= remainingBitsPerInt;+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerInt;+ if (remainingBitsPerValue == 0) {+ idx++;+ remainingBitsPerValue = bitsPerValue;+ }+ } else {+ final int mask1, mask2;+ if (primitiveSize == 8) {+ mask1 = MASKS8[remainingBitsPerValue];+ mask2 = MASKS8[remainingBitsPerInt - remainingBitsPerValue];+ } else if (primitiveSize == 16) {+ mask1 = MASKS16[remainingBitsPerValue];+ mask2 = MASKS16[remainingBitsPerInt - remainingBitsPerValue];+ } else {+ mask1 = MASKS32[remainingBitsPerValue];+ mask2 = MASKS32[remainingBitsPerInt - remainingBitsPerValue];+ }+ tmp[tmpIdx] |= (ints[idx++] & mask1) << (remainingBitsPerInt - remainingBitsPerValue);+ remainingBitsPerValue = bitsPerValue - remainingBitsPerInt + remainingBitsPerValue;+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & mask2;+ }+ }++ for (int i = 0; i < numIntsPerShift; ++i) {+ out.writeInt(tmp[i]);+ }+ }++ /** Number of bytes required to encode 256 integers of {@code bitsPerValue} bits per value. */+ static int numBytes(int bitsPerValue) {+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);+ }++ static void decodeSlow(int bitsPerValue, PostingDecodingUtil pdu, int[] tmp, int[] ints)+ throws IOException {+ final int numInts = bitsPerValue << 3;+ final int mask = MASKS32[bitsPerValue];+ pdu.splitInts(numInts, ints, 32 - bitsPerValue, 32, mask, tmp, 0, -1);+ final int remainingBitsPerInt = 32 - bitsPerValue;+ final int mask32RemainingBitsPerInt = MASKS32[remainingBitsPerInt];+ int tmpIdx = 0;+ int remainingBits = remainingBitsPerInt;+ for (int intsIdx = numInts; intsIdx < BLOCK_SIZE; ++intsIdx) {+ int b = bitsPerValue - remainingBits;+ int l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;+ while (b >= remainingBitsPerInt) {+ b -= remainingBitsPerInt;+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerInt) << b;+ }+ if (b > 0) {+ l |= (tmp[tmpIdx] >>> (remainingBitsPerInt - b)) & MASKS32[b];+ remainingBits = remainingBitsPerInt - b;+ } else {+ remainingBits = remainingBitsPerInt;+ }+ ints[intsIdx] = l;+ }+ }++"""++def writeRemainder(bpv, next_primitive, remaining_bits_per_int, o, num_values, f):+ iteration = 1+ num_ints = bpv * num_values / remaining_bits_per_int+ while num_ints % 2 == 0 and num_values % 2 == 0:+ num_ints /= 2+ num_values /= 2+ iteration *= 2+ f.write(' for (int iter = 0, tmpIdx = 0, intsIdx = %d; iter < %d; ++iter, tmpIdx += %d, intsIdx += %d) {\n' %(o, iteration, num_ints, num_values))+ i = 0+ remaining_bits = 0+ tmp_idx = 0+ for i in range(int(num_values)):+ b = bpv+ if remaining_bits == 0:+ b -= remaining_bits_per_int+ f.write(' int l%d = tmp[tmpIdx + %d] << %d;\n' %(i, tmp_idx, b))+ else:+ b -= remaining_bits+ f.write(' int l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits, b))+ tmp_idx += 1+ while b >= remaining_bits_per_int:+ b -= remaining_bits_per_int+ f.write(' l%d |= tmp[tmpIdx + %d] << %d;\n' %(i, tmp_idx, b))+ tmp_idx += 1+ if b > 0:+ f.write(' l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n' %(i, tmp_idx, remaining_bits_per_int-b, next_primitive, b))+ remaining_bits = remaining_bits_per_int-b+ f.write(' ints[intsIdx + %d] = l%d;\n' %(i, i))+ f.write(' }\n')+++def writeDecode(bpv, f):+ next_primitive = 32+ if bpv <= 8:+ next_primitive = 8+ elif bpv <= 16:+ next_primitive = 16+ if bpv == next_primitive:+ f.write(' static void decode%d(PostingDecodingUtil pdu, int[] ints) throws IOException {\n' %bpv)+ f.write(' pdu.in.readInts(ints, 0, %d);\n' %(bpv*8))+ else:+ num_values_per_int = 32 / next_primitive+ remaining_bits = next_primitive % bpv+ num_iters = (next_primitive - 1) // bpv+ o = 8 * bpv * num_iters+ if remaining_bits == 0:+ f.write(' static void decode%d(PostingDecodingUtil pdu, int[] ints) throws IOException {\n' %bpv)+ f.write(' pdu.splitInts(%d, ints, %d, %d, MASK%d_%d, ints, %d, MASK%d_%d);\n' %(bpv*8, next_primitive - bpv, bpv, next_primitive, bpv, o, next_primitive, next_primitive - num_iters * bpv))+ else:+ f.write(' static void decode%d(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {\n' %bpv)+ f.write(' pdu.splitInts(%d, ints, %d, %d, MASK%d_%d, tmp, 0, MASK%d_%d);\n' %(bpv*8, next_primitive - bpv, bpv, next_primitive, bpv, next_primitive, next_primitive - num_iters * bpv))+ writeRemainder(bpv, next_primitive, remaining_bits, o, 256/num_values_per_int - o, f)+ f.write(' }\n')++if __name__ == '__main__':+ f = open(OUTPUT_FILE, 'w')+ f.write(HEADER)+ for primitive_size in PRIMITIVE_SIZE:+ f.write(' static final int[] MASKS%d = new int[%d];\n' %(primitive_size, primitive_size))+ f.write('\n')+ f.write(' static {\n')+ for primitive_size in PRIMITIVE_SIZE:+ f.write(' for (int i = 0; i < %d; ++i) {\n' %primitive_size)+ f.write(' MASKS%d[i] = mask%d(i);\n' %(primitive_size, primitive_size))+ f.write(' }\n')+ f.write(' }')+ f.write("""+ // mark values in array as final ints to avoid the cost of reading array, arrays should only be+ // used when the idx is a variable+""")+ for primitive_size in PRIMITIVE_SIZE:+ for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)):+ f.write(' static final int MASK%d_%d = MASKS%d[%d];\n' %(primitive_size, bpv, primitive_size, bpv))++ f.write("""+ /** Decode 256 integers into {@code ints}. */+ void decode(int bitsPerValue, PostingDecodingUtil pdu, int[] ints) throws IOException {+ switch (bitsPerValue) {+""")+ for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):+ next_primitive = 32+ if bpv <= 8:+ next_primitive = 8+ elif bpv <= 16:+ next_primitive = 16+ f.write(' case %d:\n' %bpv)+ if next_primitive % bpv == 0:+ f.write(' decode%d(pdu, ints);\n' %bpv)+ else:+ f.write(' decode%d(pdu, tmp, ints);\n' %bpv)+ if next_primitive != 32:+ f.write(' expand%d(ints);\n' %next_primitive)+ f.write(' break;\n')+ f.write(' default:\n')+ f.write(' decodeSlow(bitsPerValue, pdu, tmp, ints);\n')+ f.write(' break;\n')+ f.write(' }\n')+ f.write(' }\n')++ for i in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):+ writeDecode(i, f)+ if i < MAX_SPECIALIZED_BITS_PER_VALUE:+ f.write('\n')++ f.write('}\n')diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/package-info.javasimilarity index 96%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.javarename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/package-info.javaindex 0c4bb5a1dde..ebcf8c1e5e4 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/package-info.java@@ -16,7 +16,7 @@*//**- * Lucene 10.3 file format.+ * Lucene 10.4 file format.** <h2>Apache Lucene - Index File Formats</h2>*@@ -151,15 +151,15 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term dictionary}. A+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term dictionary}. A* dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Frequency data}. For+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Frequency data}. For* each term in the dictionary, the numbers of all the documents that contain that term, and* the frequency of the term in that document, unless frequencies are omitted ({@link* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Proximity data}. For+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Proximity data}. For* each term in the dictionary, the positions that the term occurs in each document. Note that* this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>@@ -414,11 +414,13 @@* <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector* format to utilize int8 quantized vectors for float32 vector search.* <li>In version 9.12, skip data was refactored to have only two levels: every 128 docs and every- * 4,06 docs, and to be inlined in postings lists. This resulted in a speedup for queries that- * need skipping, especially conjunctions.+ * 4,096 docs, and to be inlined in postings lists. This resulted in a speedup for queries+ * that need skipping, especially conjunctions.* <li>In version 10.1, block encoding changed to be optimized for int[] storage instead of* long[].* <li>In version 10.3, the index of block tree changed to be specialized trie instead of FST.+ * <li>In version 10.4, the block size was increased from 128 to 256. There are now 256 and 8,192+ * postings between skip pointers instead of 128 and 4,096.* </ul>** <a id="Limitations"></a>@@ -433,4 +435,4 @@* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt* VInt} values which have no limit. </div>*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.javaindex 48c515b7df5..24864318af5 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java@@ -202,8 +202,8 @@ public abstract class VectorizationProvider {Set.of("org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil","org.apache.lucene.util.VectorUtil",- "org.apache.lucene.codecs.lucene103.Lucene103PostingsReader",- "org.apache.lucene.codecs.lucene103.PostingIndexInput",+ "org.apache.lucene.codecs.lucene104.Lucene104PostingsReader",+ "org.apache.lucene.codecs.lucene104.PostingIndexInput","org.apache.lucene.tests.util.TestSysoutsLimits");private static void ensureCaller() {diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.javaindex e478c08e05e..b762fced2ee 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java@@ -21,8 +21,8 @@ import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.Objects;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;import org.apache.lucene.index.ImpactsEnum;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;@@ -399,18 +399,18 @@ public class PhraseQuery extends Query {/*** A guess of the average number of simple operations for the initial seek and buffer refill per* document for the positions of a term. See also {@link- * Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}.+ * Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}.** <p>Aside: Instead of being constant this could depend among others on {@link- * Lucene103PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link+ * Lucene104PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block* size of the device storing the index.*/- private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;+ private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 256;/**- * Number of simple operations in {@link Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}+ * Number of simple operations in {@link Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}* when no seek or buffer refill is done.*/private static final int TERM_OPS_PER_POS = 7;diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.javaindex 8562ee84c01..457a2cbeefe 100644--- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java+++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java@@ -20,7 +20,7 @@ import java.io.IOException;import java.util.Comparator;import java.util.stream.LongStream;import java.util.stream.StreamSupport;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;import org.apache.lucene.index.ImpactsEnum;import org.apache.lucene.util.Bits;import org.apache.lucene.util.FixedBitSet;@@ -32,7 +32,7 @@ import org.apache.lucene.util.VectorUtil;class ScorerUtil {private static final Class<?> DEFAULT_IMPACTS_ENUM_CLASS =- Lucene103PostingsFormat.getImpactsEnumImpl();+ Lucene104PostingsFormat.getImpactsEnumImpl();private static final Class<?> DEFAULT_ACCEPT_DOCS_CLASS =new FixedBitSet(1).asReadOnlyBits().getClass();diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codecindex b85efc3c37f..c13868f1641 100644--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec@@ -13,4 +13,4 @@# See the License for the specific language governing permissions and# limitations under the License.-org.apache.lucene.codecs.lucene103.Lucene103Codec+org.apache.lucene.codecs.lucene104.Lucene104Codecdiff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex 12d44d82f7d..935a82cb22b 100644--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -13,4 +13,4 @@# See the License for the specific language governing permissions and# limitations under the License.-org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat+org.apache.lucene.codecs.lucene104.Lucene104PostingsFormatdiff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestBitSetUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestBitSetUtil.javasimilarity index 94%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestBitSetUtil.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestBitSetUtil.javaindex dbe4f9f36d8..966c61cfe28 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestBitSetUtil.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestBitSetUtil.java@@ -14,9 +14,9 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.BLOCK_SIZE;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.FixedBitSet;diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestForUtil.javasimilarity index 97%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestForUtil.javaindex 935eff6e8fd..50386ec1f8e 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestForUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;import com.carrotsearch.randomizedtesting.generators.RandomNumbers;import java.io.IOException;@@ -71,7 +71,7 @@ public class TestForUtil extends LuceneTestCase {// decodeIndexInput in = d.openInput("test.bin", IOContext.READONCE);PostingDecodingUtil pdu =- Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);+ Lucene104PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);ForUtil forUtil = new ForUtil();for (int i = 0; i < iterations; ++i) {final int bitsPerValue = in.readByte();diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104PostingsFormat.javanew file mode 100644index 00000000000..ac1a445ddd5--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104PostingsFormat.java@@ -0,0 +1,160 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collections;+import java.util.List;+import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;+import org.apache.lucene.codecs.Impact;+import org.apache.lucene.codecs.lucene103.blocktree.FieldReader;+import org.apache.lucene.codecs.lucene103.blocktree.Stats;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.FreqAndNormBuffer;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.ByteArrayDataOutput;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.index.BasePostingsFormatTestCase;+import org.apache.lucene.tests.util.TestUtil;++public class TestLucene104PostingsFormat extends BasePostingsFormatTestCase {++ @Override+ protected Codec getCodec() {+ return TestUtil.alwaysPostingsFormat(new Lucene104PostingsFormat());+ }++ public void testVInt15() throws IOException {+ byte[] bytes = new byte[5];+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);+ ByteArrayDataInput in = new ByteArrayDataInput();+ for (int i : new int[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE}) {+ out.reset(bytes);+ Lucene104PostingsWriter.writeVInt15(out, i);+ in.reset(bytes, 0, out.getPosition());+ assertEquals(i, Lucene104PostingsReader.readVInt15(in));+ assertEquals(out.getPosition(), in.getPosition());+ }+ }++ public void testVLong15() throws IOException {+ byte[] bytes = new byte[9];+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);+ ByteArrayDataInput in = new ByteArrayDataInput();+ for (long i : new long[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE, Long.MAX_VALUE}) {+ out.reset(bytes);+ Lucene104PostingsWriter.writeVLong15(out, i);+ in.reset(bytes, 0, out.getPosition());+ assertEquals(i, Lucene104PostingsReader.readVLong15(in));+ assertEquals(out.getPosition(), in.getPosition());+ }+ }++ /** Make sure the final sub-block(s) are not skipped. */+ public void testFinalBlock() throws Exception {+ Directory d = newDirectory();+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));+ for (int i = 0; i < 25; i++) {+ Document doc = new Document();+ doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));+ doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));+ w.addDocument(doc);+ }+ w.forceMerge(1);++ DirectoryReader r = DirectoryReader.open(w);+ assertEquals(1, r.leaves().size());+ FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");+ // We should see exactly two blocks: one root block (prefix empty string) and one block for z*+ // terms (prefix z):+ Stats stats = field.getStats();+ assertEquals(0, stats.floorBlockCount);+ assertEquals(2, stats.nonFloorBlockCount);+ r.close();+ w.close();+ d.close();+ }++ public void testImpactSerialization() throws IOException {+ // omit norms and omit freqs+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));++ // omit freqs+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));+ // omit freqs with very large norms+ doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));++ // omit norms+ doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));+ // omit norms with large freq+ doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));++ // freqs and norms, basic+ doTestImpactSerialization(+ Arrays.asList(+ new Impact(1, 7L),+ new Impact(3, 9L),+ new Impact(7, 10L),+ new Impact(15, 11L),+ new Impact(20, 13L),+ new Impact(28, 14L)));++ // freqs and norms, high values+ doTestImpactSerialization(+ Arrays.asList(+ new Impact(2, 2L),+ new Impact(10, 10L),+ new Impact(12, 50L),+ new Impact(50, -100L),+ new Impact(1000, -80L),+ new Impact(1005, -3L)));+ }++ private void doTestImpactSerialization(List<Impact> impacts) throws IOException {+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();+ for (Impact impact : impacts) {+ acc.add(impact.freq, impact.norm);+ }+ try (Directory dir = newDirectory()) {+ try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {+ Lucene104PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);+ }+ try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {+ byte[] b = new byte[Math.toIntExact(in.length())];+ in.readBytes(b, 0, b.length);+ FreqAndNormBuffer impactBuffer = new FreqAndNormBuffer();+ impactBuffer.growNoCopy(impacts.size() + random().nextInt(3));+ impactBuffer = Lucene104PostingsReader.readImpacts(new ByteArrayDataInput(b), impactBuffer);+ List<Impact> impacts2 = new ArrayList<>();+ for (int i = 0; i < impactBuffer.size; ++i) {+ impacts2.add(new Impact(impactBuffer.freqs[i], impactBuffer.norms[i]));+ }+ }+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPForUtil.javasimilarity index 94%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPForUtil.javaindex 61879034ffe..211b7b2427d 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPForUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;import com.carrotsearch.randomizedtesting.generators.RandomNumbers;import java.io.IOException;@@ -41,8 +41,8 @@ public class TestPForUtil extends LuceneTestCase {IndexInput in = d.openInput("test.bin", IOContext.READONCE);PostingDecodingUtil pdu =- Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);- final PForUtil pforUtil = new PForUtil();+ Lucene104PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);+ final PForUtil pforUtil = new PForUtil(new ForUtil());for (int i = 0; i < iterations; ++i) {if (random().nextInt(5) == 0) {PForUtil.skip(in);@@ -90,7 +90,7 @@ public class TestPForUtil extends LuceneTestCase {private long encodeTestData(int iterations, int[] values, Directory d) throws IOException {IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);- final PForUtil pforUtil = new PForUtil();+ final PForUtil pforUtil = new PForUtil(new ForUtil());for (int i = 0; i < iterations; ++i) {int[] source = new int[ForUtil.BLOCK_SIZE];diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPostingsUtil.javasimilarity index 98%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPostingsUtil.javaindex 32edd3608ba..5f4219a33b7 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPostingsUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene103;+package org.apache.lucene.codecs.lucene104;import java.io.IOException;import org.apache.lucene.store.Directory;diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.javaindex b41988631e6..0e9889f54c8 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java@@ -18,7 +18,7 @@ package org.apache.lucene.codecs.lucene90;import com.carrotsearch.randomizedtesting.generators.RandomPicks;import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.lucene103.Lucene103Codec;+import org.apache.lucene.codecs.lucene104.Lucene104Codec;import org.apache.lucene.document.Document;import org.apache.lucene.document.StoredField;import org.apache.lucene.index.DirectoryReader;@@ -31,7 +31,7 @@ import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {@Overrideprotected Codec getCodec() {- return new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION);+ return new Lucene104Codec(Lucene104Codec.Mode.BEST_COMPRESSION);}/**@@ -42,7 +42,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFiefor (int i = 0; i < 10; i++) {IndexWriterConfig iwc = newIndexWriterConfig();iwc.setCodec(- new Lucene103Codec(RandomPicks.randomFrom(random(), Lucene103Codec.Mode.values())));+ new Lucene104Codec(RandomPicks.randomFrom(random(), Lucene104Codec.Mode.values())));IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());Document doc = new Document();doc.add(new StoredField("field1", "value1"));@@ -72,7 +72,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFieexpectThrows(NullPointerException.class,() -> {- new Lucene103Codec(null);+ new Lucene104Codec(null);});expectThrows(diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.javaindex 0a9fb4b1ce9..8c96bdaa54d 100644--- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java+++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java@@ -16,7 +16,7 @@*/package org.apache.lucene.internal.vectorization;-import org.apache.lucene.codecs.lucene103.ForUtil;+import org.apache.lucene.codecs.lucene104.ForUtil;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;diff --git a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.javaindex ba22fad1ed0..1f6c5d525e4 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java@@ -25,8 +25,6 @@ import java.util.List;import java.util.concurrent.Executor;import java.util.concurrent.ForkJoinPool;import java.util.concurrent.ForkJoinWorkerThread;-import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103Codec;import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.KnnFloatVectorField;@@ -45,6 +43,7 @@ import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.TaskExecutor;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.VectorUtil;/** Tests reordering vector values using Binary Partitioning */@@ -64,12 +63,7 @@ public class TestBpVectorReorderer extends LuceneTestCase {private void createQuantizedIndex(Directory dir, List<float[]> vectors) throws IOException {IndexWriterConfig cfg = new IndexWriterConfig();cfg.setCodec(- new Lucene103Codec() {- @Override- public KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswScalarQuantizedVectorsFormat(8, 32);- }- });+ TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat(8, 32)));try (IndexWriter writer = new IndexWriter(dir, cfg)) {int i = 0;for (float[] vector : vectors) {diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.javaindex e075aaccc51..68616498a8f 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java@@ -21,8 +21,8 @@ import java.io.IOException;import java.util.Collection;import java.util.Collections;import java.util.Objects;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.PostingsEnum;@@ -261,18 +261,18 @@ class TermIntervalsSource extends IntervalsSource {/*** A guess of the average number of simple operations for the initial seek and buffer refill per* document for the positions of a term. See also {@link- * Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}.+ * Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}.** <p>Aside: Instead of being constant this could depend among others on {@link- * Lucene103PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link+ * Lucene104PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block* size of the device storing the index.*/- private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;+ private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 256;/**- * Number of simple operations in {@link Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}+ * Number of simple operations in {@link Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}* when no seek or buffer refill is done.*/private static final int TERM_OPS_PER_POS = 7;diff --git a/lucene/suggest/src/java/module-info.java b/lucene/suggest/src/java/module-info.javaindex 20542ac5281..c9636811965 100644--- a/lucene/suggest/src/java/module-info.java+++ b/lucene/suggest/src/java/module-info.java@@ -33,7 +33,8 @@ module org.apache.lucene.suggest {org.apache.lucene.search.suggest.document.Completion90PostingsFormat,org.apache.lucene.search.suggest.document.Completion99PostingsFormat,org.apache.lucene.search.suggest.document.Completion912PostingsFormat,- org.apache.lucene.search.suggest.document.Completion101PostingsFormat;+ org.apache.lucene.search.suggest.document.Completion101PostingsFormat,+ org.apache.lucene.search.suggest.document.Completion104PostingsFormat;provides org.apache.lucene.analysis.TokenFilterFactory withorg.apache.lucene.search.suggest.analyzing.SuggestStopFilterFactory;}diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.javaindex 5e419016945..2ac980fe537 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java@@ -19,8 +19,10 @@ package org.apache.lucene.search.suggest.document;import org.apache.lucene.codecs.PostingsFormat;/**- * {@link CompletionPostingsFormat} for {@link- * org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat}+ * {@link CompletionPostingsFormat} for {@code+ * org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat}. This format is only used+ * for backward-compatibility of the index format and cannot be used to write data, use {@link+ * Completion104PostingsFormat} on new indices.** @lucene.experimental*/diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion104PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion104PostingsFormat.javanew file mode 100644index 00000000000..69b49cf7d7d--- /dev/null+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion104PostingsFormat.java@@ -0,0 +1,37 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search.suggest.document;++import org.apache.lucene.codecs.PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;++/**+ * {@link CompletionPostingsFormat} for {@link Lucene104PostingsFormat}.+ *+ * @lucene.experimental+ */+public class Completion104PostingsFormat extends CompletionPostingsFormat {+ /** Creates a {@link Completion104PostingsFormat}. */+ public Completion104PostingsFormat() {+ super("Completion104");+ }++ @Override+ protected PostingsFormat delegatePostingsFormat() {+ return PostingsFormat.forName("Lucene104");+ }+}diff --git a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex 5c0365616cd..fa4a80aaf67 100644--- a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -36,3 +36,4 @@ org.apache.lucene.search.suggest.document.Completion90PostingsFormatorg.apache.lucene.search.suggest.document.Completion99PostingsFormatorg.apache.lucene.search.suggest.document.Completion912PostingsFormatorg.apache.lucene.search.suggest.document.Completion101PostingsFormat+org.apache.lucene.search.suggest.document.Completion104PostingsFormatdiff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.javaindex 1990baffe72..1548972c814 100644--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java@@ -948,7 +948,7 @@ public class TestSuggestField extends LuceneTestCase {iwc.setMergePolicy(newLogMergePolicy());Codec filterCodec =new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {- final PostingsFormat postingsFormat = new Completion101PostingsFormat();+ final PostingsFormat postingsFormat = new Completion104PostingsFormat();@Overridepublic PostingsFormat postingsFormat() {diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.javaindex 17ceff0d4c2..70e24119da4 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java@@ -28,9 +28,9 @@ import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -38,7 +38,7 @@ import org.apache.lucene.index.SegmentWriteState;// any PostingsFormat and make it ord-able.../**- * Customized version of {@link Lucene103PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.+ * Customized version of {@link Lucene104PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.*/public final class LuceneFixedGap extends PostingsFormat {final int termIndexInterval;@@ -54,7 +54,7 @@ public final class LuceneFixedGap extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase docs = new Lucene103PostingsWriter(state);+ PostingsWriterBase docs = new Lucene104PostingsWriter(state);// TODO: should we make the terms index more easily// pluggable? Ie so that this codec would record which@@ -91,7 +91,7 @@ public final class LuceneFixedGap extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postings = new Lucene103PostingsReader(state);+ PostingsReaderBase postings = new Lucene104PostingsReader(state);TermsIndexReaderBase indexReader;boolean success = false;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.javaindex 741ce732518..719e698eda8 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;// any PostingsFormat and make it ord-able.../**- * Customized version of {@link Lucene103PostingsFormat} that uses {@link+ * Customized version of {@link Lucene104PostingsFormat} that uses {@link* VariableGapTermsIndexWriter} with a fixed interval, but forcing high docfreq terms to be indexed* terms.*/@@ -59,7 +59,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase docs = new Lucene103PostingsWriter(state);+ PostingsWriterBase docs = new Lucene104PostingsWriter(state);// TODO: should we make the terms index more easily// pluggable? Ie so that this codec would record which@@ -100,7 +100,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postings = new Lucene103PostingsReader(state);+ PostingsReaderBase postings = new Lucene104PostingsReader(state);TermsIndexReaderBase indexReader;boolean success = false;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.javaindex 93dd141b061..9de1e12c427 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;// any PostingsFormat and make it ord-able.../**- * Customized version of {@link Lucene103PostingsFormat} that uses {@link+ * Customized version of {@link Lucene104PostingsFormat} that uses {@link* VariableGapTermsIndexWriter} with a fixed interval.*/public final class LuceneVarGapFixedInterval extends PostingsFormat {@@ -56,7 +56,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase docs = new Lucene103PostingsWriter(state);+ PostingsWriterBase docs = new Lucene104PostingsWriter(state);// TODO: should we make the terms index more easily// pluggable? Ie so that this codec would record which@@ -95,7 +95,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postings = new Lucene103PostingsReader(state);+ PostingsReaderBase postings = new Lucene104PostingsReader(state);TermsIndexReaderBase indexReader;boolean success = false;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.javaindex e660ccc4478..0c1fa8b4f00 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java@@ -35,10 +35,10 @@ import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.codecs.memory.FSTTermsReader;import org.apache.lucene.codecs.memory.FSTTermsWriter;import org.apache.lucene.index.FieldInfo;@@ -121,7 +121,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {random.nextInt(); // consume a random for buffersize- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);final FieldsConsumer fields;final int t1 = random.nextInt(4);@@ -289,7 +289,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);}- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);final FieldsProducer fields;final int t1 = random.nextInt(4);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.javaindex 63f7bcbf2cb..f6abfe2caf6 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;import org.apache.lucene.codecs.uniformsplit.BlockDecoder;import org.apache.lucene.codecs.uniformsplit.BlockEncoder;import org.apache.lucene.codecs.uniformsplit.IndexDictionary;@@ -68,7 +68,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState segmentWriteState) throws IOException {- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(segmentWriteState);+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(segmentWriteState);boolean success = false;try {FieldsConsumer fieldsConsumer = createFieldsConsumer(segmentWriteState, postingsWriter);@@ -146,7 +146,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState segmentReadState) throws IOException {- PostingsReaderBase postingsReader = new Lucene103PostingsReader(segmentReadState);+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(segmentReadState);boolean success = false;try {FieldsProducer fieldsProducer = createFieldsProducer(segmentReadState, postingsReader);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.javaindex 13db0d34e78..e4c0625e7c5 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java@@ -38,7 +38,7 @@ import java.util.TimeZone;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.DocValuesFormat;import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103Codec;+import org.apache.lucene.codecs.lucene104.Lucene104Codec;import org.apache.lucene.codecs.simpletext.SimpleTextCodec;import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.tests.codecs.asserting.AssertingCodec;@@ -188,9 +188,9 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule {} else if ("Compressing".equals(TEST_CODEC)|| ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {codec = CompressingCodec.randomInstance(random);- } else if ("Lucene100".equals(TEST_CODEC)- || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene100"))) {- codec = new Lucene103Codec(RandomPicks.randomFrom(random, Lucene103Codec.Mode.values()));+ } else if ("Lucene104".equals(TEST_CODEC)+ || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene104"))) {+ codec = new Lucene104Codec(RandomPicks.randomFrom(random, Lucene104Codec.Mode.values()));} else if (!"random".equals(TEST_CODEC)) {codec = Codec.forName(TEST_CODEC);} else if ("random".equals(TEST_POSTINGSFORMAT)) {diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.javaindex 1469ee02bb0..073da0d3a5b 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java@@ -62,8 +62,8 @@ import org.apache.lucene.codecs.DocValuesFormat;import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;-import org.apache.lucene.codecs.lucene103.Lucene103Codec;-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;+import org.apache.lucene.codecs.lucene104.Lucene104Codec;+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;@@ -1327,7 +1327,7 @@ public final class TestUtil {* different from {@link Codec#getDefault()} because that is randomized.*/public static Codec getDefaultCodec() {- return new Lucene103Codec();+ return new Lucene104Codec();}/**@@ -1335,7 +1335,7 @@ public final class TestUtil {* Lucene.*/public static PostingsFormat getDefaultPostingsFormat() {- return new Lucene103PostingsFormat();+ return new Lucene104PostingsFormat();}/**@@ -1346,7 +1346,7 @@ public final class TestUtil {*/public static PostingsFormat getDefaultPostingsFormat(int minItemsPerBlock, int maxItemsPerBlock) {- return new Lucene103PostingsFormat(minItemsPerBlock, maxItemsPerBlock);+ return new Lucene104PostingsFormat(minItemsPerBlock, maxItemsPerBlock);}/** Returns a random postings format that supports term ordinals */