Skip to content

Apache Lucene open-source search software

Bump block size of postings to 256. (#15160)

Adrien Grand00f2b01e7985
diff --git a/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle b/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle
index 8facd4e438d..ec90f270bba 100644
--- a/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle
+++ b/build-tools/build-infra/src/main/groovy/lucene.regenerate.forUtil.gradle
@@ -47,9 +47,7 @@ void addGenerateFromPythonTask(Project project, File genDir, String taskName, St
configure(project(":lucene:core")) {
addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/codecs/lucene103"), "ForUtil", "gen_ForUtil.py", "ForUtil.java")
- addGenerateFromPythonTask(project,
- file("src/java/org/apache/lucene/codecs/lucene103"), "ForDeltaUtil", "gen_ForDeltaUtil.py", "ForDeltaUtil.java")
+ file("src/java/org/apache/lucene/codecs/lucene104"), "ForUtil", "gen_ForUtil.py", "ForUtil.java")
}
configure(project(":lucene:backward-codecs")) {
@@ -71,4 +69,9 @@ configure(project(":lucene:backward-codecs")) {
file("src/java/org/apache/lucene/backward_codecs/lucene101"), "ForUtil101", "gen_ForUtil.py", "ForUtil.java")
addGenerateFromPythonTask(project,
file("src/java/org/apache/lucene/backward_codecs/lucene101"), "ForDeltaUtil101", "gen_ForDeltaUtil.py", "ForDeltaUtil.java")
+
+ addGenerateFromPythonTask(project,
+ file("src/java/org/apache/lucene/backward_codecs/lucene103"), "ForUtil103", "gen_ForUtil.py", "ForUtil.java")
+ addGenerateFromPythonTask(project,
+ file("src/java/org/apache/lucene/backward_codecs/lucene103"), "ForDeltaUtil103", "gen_ForDeltaUtil.py", "ForDeltaUtil.java")
}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0ce8274e2b2..1c1b6450494 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -136,6 +136,9 @@ Optimizations
* GITHUB#15151: Use `SimScorer#score` bulk API to compute impact scores per
block of postings. (Adrien Grand)
+* GITHUB#15160: Increased the size used for blocks of postings from 128 to 256.
+ This gives a noticeable speedup to many queries. (Adrien Grand)
+
Bug Fixes
---------------------
* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentException
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil103.json b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil103.json
new file mode 100644
index 00000000000..abe1d457c93
--- /dev/null
+++ b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil103.json
@@ -0,0 +1,4 @@
+{
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.java": "4cd0564cb34801443d050065ab7ae8d93e2dca92",
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.py": "8d656f27945b5a679babc41ac47afbd351359aa8"
+}
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil103.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil103.json
new file mode 100644
index 00000000000..cc5f05caa51
--- /dev/null
+++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil103.json
@@ -0,0 +1,4 @@
+{
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.java": "10904a43be9537b6c52ec13a1139c71c11345c97",
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.py": "61a6c7d73b0e8b166ebf7344fc24e95873240e9f"
+}
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 88cbd590951..5c8cf801669 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -40,6 +40,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs.lucene912;
exports org.apache.lucene.backward_codecs.lucene100;
exports org.apache.lucene.backward_codecs.lucene101;
+ exports org.apache.lucene.backward_codecs.lucene103;
exports org.apache.lucene.backward_codecs.packed;
exports org.apache.lucene.backward_codecs.store;
@@ -51,7 +52,8 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,
- org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat;
+ org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat,
+ org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
@@ -71,5 +73,6 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,
org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,
org.apache.lucene.backward_codecs.lucene100.Lucene100Codec,
- org.apache.lucene.backward_codecs.lucene101.Lucene101Codec;
+ org.apache.lucene.backward_codecs.lucene101.Lucene101Codec,
+ org.apache.lucene.backward_codecs.lucene103.Lucene103Codec;
}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/BitSetUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/BitSetUtil.java
new file mode 100644
index 00000000000..fcbeca1ae69
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/BitSetUtil.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import java.util.Objects;
+import org.apache.lucene.util.FixedBitSet;
+
+class BitSetUtil {
+
+ /**
+ * Converts set bits in the given bitset to an array of document IDs. Only processes bits from
+ * index {@code from} (inclusive) to {@code to} (exclusive) and returns the number of bits set in
+ * this range.
+ *
+ * <p>Each set bit's position is converted to a document ID by adding the {@code base} value and
+ * stored in the provided {@code array}.
+ *
+ * <p>NOTE: Caller need to ensure the {@code array} has a length greater than or equal to {@code
+ * bitSet.cardinality(from, to) + 1}.
+ */
+ static int denseBitsetToArray(FixedBitSet bitSet, int from, int to, int base, int[] array) {
+ assert bitSet.cardinality(from, to) + 1 <= array.length
+ : "Array length must be at least bitSet.cardinality(from, to) + 1";
+
+ Objects.checkFromToIndex(from, to, bitSet.length());
+
+ int offset = 0;
+ long[] bits = bitSet.getBits();
+ // First, align `from` with a word start, ie. a multiple of Long.SIZE (64)
+ if ((from & 0x3F) != 0) {
+ long word = bits[from >> 6] >>> from;
+ int numBitsTilNextWord = -from & 0x3F;
+ if (to - from < numBitsTilNextWord) {
+ // All bits are in a single word
+ word &= (1L << (to - from)) - 1L;
+ return word2Array(word, from + base, array, offset);
+ }
+ offset = word2Array(word, from + base, array, offset);
+ from += numBitsTilNextWord;
+ assert (from & 0x3F) == 0;
+ }
+
+ for (int i = from >> 6, end = to >> 6; i < end; ++i) {
+ long word = bits[i];
+ offset = word2Array(word, base + (i << 6), array, offset);
+ }
+
+ // Now handle remaining bits in the last partial word
+ if ((to & 0x3F) != 0) {
+ long word = bits[to >> 6] & ((1L << to) - 1);
+ offset = word2Array(word, base + (to & ~0x3F), array, offset);
+ }
+
+ return offset;
+ }
+
+ private static int word2Array(long word, int base, int[] docs, int offset) {
+ final int bitCount = Long.bitCount(word);
+
+ if (bitCount >= 32) {
+ return denseWord2Array(word, base, docs, offset);
+ }
+
+ for (int i = 0; i < bitCount; i++) {
+ int ntz = Long.numberOfTrailingZeros(word);
+ docs[offset++] = base + ntz;
+ word ^= 1L << ntz;
+ }
+
+ return offset;
+ }
+
+ private static int denseWord2Array(long word, int base, int[] docs, int offset) {
+ final int lWord = (int) word;
+ final int hWord = (int) (word >>> 32);
+ final int offset32 = offset + Integer.bitCount(lWord);
+ int hOffset = offset32;
+
+ for (int i = 0; i < 32; i++) {
+ docs[offset] = base + i;
+ docs[hOffset] = base + i + 32;
+ offset += (lWord >>> i) & 1;
+ hOffset += (hWord >>> i) & 1;
+ }
+
+ docs[offset32] = base + 32 + Integer.numberOfTrailingZeros(hWord);
+
+ return hOffset;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.java
similarity index 82%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.java
index e577ce81aaf..481f04de566 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForDeltaUtil.java
@@ -16,46 +16,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
-import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_1;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_2;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_4;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_5;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_6;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_7;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_8;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_1;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_10;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_11;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_12;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_13;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_14;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_15;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_16;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_2;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_3;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_4;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_5;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_6;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_7;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_8;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_9;
-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse16;
-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse8;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode1;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode10;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode2;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode3;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode9;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decodeSlow;
-import static org.apache.lucene.codecs.lucene103.ForUtil.encode;
-import static org.apache.lucene.codecs.lucene103.ForUtil.expand16;
-import static org.apache.lucene.codecs.lucene103.ForUtil.expand8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_1;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_2;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_4;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_5;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_6;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_7;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_1;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_10;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_11;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_12;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_13;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_14;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_15;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_16;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_2;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_3;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_4;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_5;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_6;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_7;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_9;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse16;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode1;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode10;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode2;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode3;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode9;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decodeSlow;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.encode;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand16;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand8;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.packed.PackedInts;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.java
index 4ccfbd28125..4e3a8a83b57 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/ForUtil.java
@@ -16,10 +16,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
/**
@@ -29,7 +28,9 @@ import org.apache.lucene.store.DataOutput;
*/
public final class ForUtil {
+ /** Number of integers per block. */
public static final int BLOCK_SIZE = 128;
+
static final int BLOCK_SIZE_LOG2 = 7;
static int expandMask16(int mask16) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103Codec.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103Codec.java
index d2e3eea6ef9..b571c12a5df 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103Codec.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.util.Objects;
import org.apache.lucene.codecs.Codec;
@@ -49,7 +49,7 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
*
* <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
*
- * @see org.apache.lucene.codecs.lucene103 package documentation for file format details.
+ * @see org.apache.lucene.backward_codecs.lucene103 package documentation for file format details.
* @lucene.experimental
*/
public class Lucene103Codec extends Codec {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsFormat.java
similarity index 94%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsFormat.java
index 04880504311..9e9c8d3e7fc 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;
import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
-import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -319,7 +318,7 @@ import org.apache.lucene.util.packed.PackedInts;
*
* @lucene.experimental
*/
-public final class Lucene103PostingsFormat extends PostingsFormat {
+public class Lucene103PostingsFormat extends PostingsFormat {
/** Filename extension for some small metadata about how postings are encoded. */
public static final String META_EXTENSION = "psm";
@@ -342,7 +341,7 @@ public final class Lucene103PostingsFormat extends PostingsFormat {
/** Size of blocks. */
public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
- public static final int BLOCK_MASK = BLOCK_SIZE - 1;
+ static final int BLOCK_MASK = BLOCK_SIZE - 1;
/** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */
public static final int LEVEL1_FACTOR = 32;
@@ -350,17 +349,7 @@ public final class Lucene103PostingsFormat extends PostingsFormat {
/** Total number of docs covered by level 1 skip data: 32 * 128 = 4,096 */
public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;
- public static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
-
- /**
- * Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is
- * internally used to help the JVM make good inlining decisions.
- *
- * @lucene.internal
- */
- public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {
- return Lucene103PostingsReader.BlockPostingsEnum.class;
- }
+ static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
static final String TERMS_CODEC = "Lucene103PostingsWriterTerms";
static final String META_CODEC = "Lucene103PostingsWriterMeta";
@@ -372,10 +361,6 @@ public final class Lucene103PostingsFormat extends PostingsFormat {
static final int VERSION_CURRENT = VERSION_START;
- private final int version;
- private final int minTermBlockSize;
- private final int maxTermBlockSize;
-
/** Creates {@code Lucene103PostingsFormat} with default settings. */
public Lucene103PostingsFormat() {
this(
@@ -391,31 +376,13 @@ public final class Lucene103PostingsFormat extends PostingsFormat {
* Lucene103BlockTreeTermsWriter#Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
*/
public Lucene103PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
- this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
- }
-
- /** Expert constructor that allows setting the version. */
- public Lucene103PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
super("Lucene103");
- if (version < VERSION_START || version > VERSION_CURRENT) {
- throw new IllegalArgumentException("Version out of range: " + version);
- }
- this.version = version;
- Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
- this.minTermBlockSize = minTermBlockSize;
- this.maxTermBlockSize = maxTermBlockSize;
}
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state, version);
- try {
- return new Lucene103BlockTreeTermsWriter(
- state, postingsWriter, minTermBlockSize, maxTermBlockSize);
- } catch (Throwable t) {
- IOUtils.closeWhileSuppressingExceptions(t, postingsWriter);
- throw t;
- }
+ throw new UnsupportedOperationException(
+ "This postings format may not be used for writing, use the current postings format");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.java
similarity index 96%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.java
index 1d5c71167ed..84d50d551cd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.java
@@ -14,24 +14,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
-
-import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.LEVEL1_NUM_DOCS;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.META_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.VERSION_START;
+package org.apache.lucene.backward_codecs.lucene103;
+
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.LEVEL1_NUM_DOCS;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.META_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.VERSION_START;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FreqAndNormBuffer;
import org.apache.lucene.index.Impacts;
@@ -40,8 +40,6 @@ import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
-import org.apache.lucene.internal.vectorization.VectorizationProvider;
import org.apache.lucene.search.DocAndFloatFeatureBuffer;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -63,8 +61,6 @@ import org.apache.lucene.util.VectorUtil;
*/
public final class Lucene103PostingsReader extends PostingsReaderBase {
- static final VectorizationProvider VECTORIZATION_PROVIDER = VectorizationProvider.getInstance();
-
private final IndexInput docIn;
private final IndexInput posIn;
private final IndexInput payIn;
@@ -452,7 +448,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
if (needsPos) {
this.posIn = Lucene103PostingsReader.this.posIn.clone();
- posInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(posIn);
+ posInUtil = new PostingDecodingUtil(posIn);
posDeltaBuffer = new int[BLOCK_SIZE];
} else {
this.posIn = null;
@@ -462,7 +458,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
if (needsOffsets || needsPayloads) {
this.payIn = Lucene103PostingsReader.this.payIn.clone();
- payInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(payIn);
+ payInUtil = new PostingDecodingUtil(payIn);
} else {
this.payIn = null;
payInUtil = null;
@@ -505,7 +501,7 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
if (docIn == null) {
// lazy init
docIn = Lucene103PostingsReader.this.docIn.clone();
- docInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(docIn);
+ docInUtil = new PostingDecodingUtil(docIn);
}
prefetchPostings(docIn, termState);
}
@@ -1395,9 +1391,6 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
}
}
- /**
- * @see Lucene103PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)
- */
static int readVInt15(DataInput in) throws IOException {
short s = in.readShort();
if (s >= 0) {
@@ -1407,9 +1400,6 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
}
}
- /**
- * @see Lucene103PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)
- */
static long readVLong15(DataInput in) throws IOException {
short s = in.readShort();
if (s >= 0) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PForUtil.java
similarity index 97%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PForUtil.java
index 6c8600a6e81..e7d8e0300ee 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PForUtil.java
@@ -14,11 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.packed.PackedInts;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingDecodingUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingDecodingUtil.java
new file mode 100644
index 00000000000..99f3c01f44d
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingDecodingUtil.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexInput;
+
+/** Utility class to decode postings. */
+final class PostingDecodingUtil {
+
+ /** The wrapper {@link IndexInput}. */
+ public final IndexInput in;
+
+ /** Sole constructor, called by sub-classes. */
+ PostingDecodingUtil(IndexInput in) {
+ this.in = in;
+ }
+
+ /**
+ * Core methods for decoding blocks of docs / freqs / positions / offsets.
+ *
+ * <ul>
+ * <li>Read {@code count} ints.
+ * <li>For all {@code i} &gt;= 0 so that {@code bShift - i * dec} &gt; 0, apply shift {@code
+ * bShift - i * dec} and store the result in {@code b} at offset {@code count * i}.
+ * <li>Apply mask {@code cMask} and store the result in {@code c} starting at offset {@code
+ * cIndex}.
+ * </ul>
+ */
+ public void splitInts(
+ int count, int[] b, int bShift, int dec, int bMask, int[] c, int cIndex, int cMask)
+ throws IOException {
+ in.readInts(c, cIndex, count);
+ final int maxIter = (bShift - 1) / dec;
+
+ // Process each shift level across all elements (better for vectorization)
+ for (int j = 0; j <= maxIter; ++j) {
+ final int shift = bShift - j * dec;
+ final int bOffset = count * j;
+ // Vectorizable loop: contiguous memory access with simple operations
+ for (int i = 0; i < count; ++i) {
+ b[bOffset + i] = (c[cIndex + i] >>> shift) & bMask;
+ }
+ }
+
+ // Apply mask to c array (vectorizable)
+ for (int i = 0; i < count; ++i) {
+ c[cIndex + i] &= cMask;
+ }
+ }
+}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingsUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingsUtil.java
new file mode 100644
index 00000000000..1038e32a93c
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/PostingsUtil.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.GroupVIntUtil;
+
+/** Utility class to encode/decode postings block. */
+final class PostingsUtil {
+
+ /**
+ * Read values that have been written using variable-length encoding and group-varint encoding
+ * instead of bit-packing.
+ */
+ static void readVIntBlock(
+ IndexInput docIn,
+ int[] docBuffer,
+ int[] freqBuffer,
+ int num,
+ boolean indexHasFreq,
+ boolean decodeFreq)
+ throws IOException {
+ GroupVIntUtil.readGroupVInts(docIn, docBuffer, num);
+ if (indexHasFreq && decodeFreq) {
+ for (int i = 0; i < num; ++i) {
+ freqBuffer[i] = docBuffer[i] & 0x01;
+ docBuffer[i] >>>= 1;
+ if (freqBuffer[i] == 0) {
+ freqBuffer[i] = docIn.readVInt();
+ }
+ }
+ } else if (indexHasFreq) {
+ for (int i = 0; i < num; ++i) {
+ docBuffer[i] >>>= 1;
+ }
+ }
+ }
+
+ /** Write freq buffer with variable-length encoding and doc buffer with group-varint encoding. */
+ static void writeVIntBlock(
+ DataOutput docOut, int[] docBuffer, int[] freqBuffer, int num, boolean writeFreqs)
+ throws IOException {
+ if (writeFreqs) {
+ for (int i = 0; i < num; i++) {
+ docBuffer[i] = (docBuffer[i] << 1) | (freqBuffer[i] == 1 ? 1 : 0);
+ }
+ }
+ docOut.writeGroupVInts(docBuffer, num);
+ if (writeFreqs) {
+ for (int i = 0; i < num; i++) {
+ final int freq = freqBuffer[i];
+ if (freq != 1) {
+ docOut.writeVInt(freq);
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.py
similarity index 78%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.py
index 290a6a29e23..912f18e6ce7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForDeltaUtil.py
@@ -40,48 +40,47 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_1;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_2;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_4;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_5;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_6;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_7;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK16_8;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_1;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_10;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_11;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_12;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_13;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_14;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_15;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_16;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_2;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_3;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_4;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_5;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_6;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_7;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_8;
-import static org.apache.lucene.codecs.lucene103.ForUtil.MASK32_9;
-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse16;
-import static org.apache.lucene.codecs.lucene103.ForUtil.collapse8;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode1;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode10;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode2;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode3;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decode9;
-import static org.apache.lucene.codecs.lucene103.ForUtil.decodeSlow;
-import static org.apache.lucene.codecs.lucene103.ForUtil.encode;
-import static org.apache.lucene.codecs.lucene103.ForUtil.expand16;
-import static org.apache.lucene.codecs.lucene103.ForUtil.expand8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_1;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_2;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_4;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_5;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_6;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_7;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK16_8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_1;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_10;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_11;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_12;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_13;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_14;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_15;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_16;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_2;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_3;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_4;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_5;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_6;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_7;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.MASK32_9;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse16;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.collapse8;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode1;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode10;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode2;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode3;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decode9;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.decodeSlow;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.encode;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand16;
+import static org.apache.lucene.backward_codecs.lucene103.ForUtil.expand8;
/**
* Inspired from https://fulmicoton.com/posts/bitpacking/
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.py
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.py
index f3bb9fb50cd..6ecfeee16d0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/gen_ForUtil.py
@@ -40,10 +40,9 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
/**
@@ -55,6 +54,7 @@ import org.apache.lucene.store.DataOutput;
*/
public final class ForUtil {
+ /** Number of integers per block. */
public static final int BLOCK_SIZE = 128;
static final int BLOCK_SIZE_LOG2 = 7;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/package-info.java
new file mode 100644
index 00000000000..f23a48fcccf
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/package-info.java
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Lucene 10.3 file format.
+ *
+ * <h2>Apache Lucene - Index File Formats</h2>
+ *
+ * <div>
+ *
+ * <ul>
+ * <li><a href="#Introduction">Introduction</a>
+ * <li><a href="#Definitions">Definitions</a>
+ * <ul>
+ * <li><a href="#Inverted_Indexing">Inverted Indexing</a>
+ * <li><a href="#Types_of_Fields">Types of Fields</a>
+ * <li><a href="#Segments">Segments</a>
+ * <li><a href="#Document_Numbers">Document Numbers</a>
+ * </ul>
+ * <li><a href="#Overview">Index Structure Overview</a>
+ * <li><a href="#File_Naming">File Naming</a>
+ * <li><a href="#file-names">Summary of File Extensions</a>
+ * <ul>
+ * <li><a href="#Lock_File">Lock File</a>
+ * <li><a href="#History">History</a>
+ * <li><a href="#Limitations">Limitations</a>
+ * </ul>
+ * </ul>
+ *
+ * </div> <a id="Introduction"></a>
+ *
+ * <h3>Introduction</h3>
+ *
+ * <div>
+ *
+ * <p>This document defines the index file formats used in this version of Lucene. If you are using
+ * a different version of Lucene, please consult the copy of <code>docs/</code> that was distributed
+ * with the version you are using.
+ *
+ * <p>This document attempts to provide a high-level definition of the Apache Lucene file formats.
+ * </div> <a id="Definitions"></a>
+ *
+ * <h3>Definitions</h3>
+ *
+ * <div>
+ *
+ * <p>The fundamental concepts in Lucene are index, document, field and term.
+ *
+ * <p>An index contains a sequence of documents.
+ *
+ * <ul>
+ * <li>A document is a sequence of fields.
+ * <li>A field is a named sequence of terms.
+ * <li>A term is a sequence of bytes.
+ * </ul>
+ *
+ * <p>The same sequence of bytes in two different fields is considered a different term. Thus terms
+ * are represented as a pair: the string naming the field, and the bytes within the field. <a
+ * id="Inverted_Indexing"></a>
+ *
+ * <h4>Inverted Indexing</h4>
+ *
+ * <p>Lucene's index stores terms and statistics about those terms in order to make term-based
+ * search more efficient. Lucene's terms index falls into the family of indexes known as an
+ * <i>inverted index.</i> This is because it can list, for a term, the documents that contain it.
+ * This is the inverse of the natural relationship, in which documents list terms. <a
+ * id="Types_of_Fields"></a>
+ *
+ * <h4>Types of Fields</h4>
+ *
+ * <p>In Lucene, fields may be <i>stored</i>, in which case their text is stored in the index
+ * literally, in a non-inverted manner. Fields that are inverted are called <i>indexed</i>. A field
+ * may be both stored and indexed.
+ *
+ * <p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the text of a field
+ * may be used literally as a term to be indexed. Most fields are tokenized, but sometimes it is
+ * useful for certain identifier fields to be indexed literally.
+ *
+ * <p>See the {@link org.apache.lucene.document.Field Field} java docs for more information on
+ * Fields. <a id="Segments"></a>
+ *
+ * <h4>Segments</h4>
+ *
+ * <p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>. Each segment is a
+ * fully independent index, which could be searched separately. Indexes evolve by:
+ *
+ * <ol>
+ * <li>Creating new segments for newly added documents.
+ * <li>Merging existing segments.
+ * </ol>
+ *
+ * <p>Searches may involve multiple segments and/or multiple indexes, each index potentially
+ * composed of a set of segments. <a id="Document_Numbers"></a>
+ *
+ * <h4>Document Numbers</h4>
+ *
+ * <p>Internally, Lucene refers to documents by an integer <i>document number</i>. The first
+ * document added to an index is numbered zero, and each subsequent document added gets a number one
+ * greater than the previous.
+ *
+ * <p>Note that a document's number may change, so caution should be taken when storing these
+ * numbers outside of Lucene. In particular, numbers may change in the following situations:
+ *
+ * <ul>
+ * <li>
+ * <p>The numbers stored in each segment are unique only within the segment, and must be
+ * converted before they can be used in a larger context. The standard technique is to
+ * allocate each segment a range of values, based on the range of numbers used in that
+ * segment. To convert a document number from a segment to an external value, the segment's
+ * <i>base</i> document number is added. To convert an external value back to a
+ * segment-specific value, the segment is identified by the range that the external value is
+ * in, and the segment's base value is subtracted. For example two five document segments
+ * might be combined, so that the first segment has a base value of zero, and the second of
+ * five. Document three from the second segment would have an external value of eight.
+ * <li>
+ * <p>When documents are deleted, gaps are created in the numbering. These are eventually
+ * removed as the index evolves through merging. Deleted documents are dropped when segments
+ * are merged. A freshly-merged segment thus has no gaps in its numbering.
+ * </ul>
+ *
+ * </div> <a id="Overview"></a>
+ *
+ * <h3>Index Structure Overview</h3>
+ *
+ * <div>
+ *
+ * <p>Each segment index maintains the following:
+ *
+ * <ul>
+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment info}. This
+ * contains metadata about a segment, such as the number of documents, what files it uses, and
+ * information about how the segment is sorted
+ * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Field names}. This
+ * contains metadata about the set of named fields used in the index.
+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Stored Field values}.
+ * This contains, for each document, a list of attribute-value pairs, where the attributes are
+ * field names. These are used to store auxiliary information about the document, such as its
+ * title, url, or an identifier to access a database. The set of stored fields are what is
+ * returned for each hit when searching. This is keyed by document number.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term
+ * dictionary}. A dictionary containing all of the terms used in all of the indexed fields of
+ * all of the documents. The dictionary also contains the number of documents which contain
+ * the term, and pointers to the term's frequency and proximity data.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Frequency
+ * data}. For each term in the dictionary, the numbers of all the documents that contain that
+ * term, and the frequency of the term in that document, unless frequencies are omitted
+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
+ * <li>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Proximity
+ * data}. For each term in the dictionary, the positions that the term occurs in each
+ * document. Note that this will not exist if all fields in all documents omit position data.
+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
+ * each field in each document, a value is stored that is multiplied into the score for hits
+ * on that field.
+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vectors}. For each
+ * field in each document, the term vector (sometimes called document vector) may be stored. A
+ * term vector consists of term text and term frequency. To add Term Vectors to your index see
+ * the {@link org.apache.lucene.document.Field Field} constructors
+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-document values}. Like
+ * stored values, these are also keyed by document number, but are generally intended to be
+ * loaded into main memory for fast access. Whereas stored values are generally intended for
+ * summary results from searches, per-document values are useful for things like scoring
+ * factors.
+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live documents}. An
+ * optional file indicating which documents are live.
+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}. Optional pair
+ * of files, recording dimensionally indexed fields, to enable fast numeric range filtering
+ * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape
+ * intersection (2D, 3D).
+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}. The
+ * vector format stores numeric vectors in a format optimized for random access and
+ * computation, supporting high-dimensional nearest-neighbor search.
+ * </ul>
+ *
+ * <p>Details on each of these are provided in their linked pages. </div> <a id="File_Naming"></a>
+ *
+ * <h3>File Naming</h3>
+ *
+ * <div>
+ *
+ * <p>All files belonging to a segment have the same name with varying extensions. The extensions
+ * correspond to the different file formats described below. When using the Compound File format
+ * (default for small segments) these files (except for the Segment info file, the Lock file, and
+ * Deleted documents file) are collapsed into a single .cfs file (see below for details)
+ *
+ * <p>Typically, all segments in an index are stored in a single directory, although this is not
+ * required.
+ *
+ * <p>File names are never re-used. That is, when any file is saved to the Directory it is given a
+ * never before used filename. This is achieved using a simple generations approach. For example,
+ * the first segments file is segments_1, then segments_2, etc. The generation is a sequential long
+ * integer represented in alpha-numeric (base 36) form. </div> <a id="file-names"></a>
+ *
+ * <h3>Summary of File Extensions</h3>
+ *
+ * <div>
+ *
+ * <p>The following table summarizes the names and extensions of the files in Lucene:
+ *
+ * <table class="padding4" style="border-spacing: 1px; border-collapse: separate">
+ * <caption>lucene filenames by extension</caption>
+ * <tr>
+ * <th>Name</th>
+ * <th>Extension</th>
+ * <th>Brief Description</th>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
+ * <td>segments_N</td>
+ * <td>Stores information about a commit point</td>
+ * </tr>
+ * <tr>
+ * <td><a href="#Lock_File">Lock File</a></td>
+ * <td>write.lock</td>
+ * <td>The Write lock prevents multiple IndexWriters from writing to the same
+ * file.</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment Info}</td>
+ * <td>.si</td>
+ * <td>Stores metadata about a segment</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat Compound File}</td>
+ * <td>.cfs, .cfe</td>
+ * <td>An optional "virtual" file consisting of all the other index files for
+ * systems that frequently run out of file handles.</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Fields}</td>
+ * <td>.fnm</td>
+ * <td>Stores information about the fields</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Index}</td>
+ * <td>.fdx</td>
+ * <td>Contains pointers to field data</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Data}</td>
+ * <td>.fdt</td>
+ * <td>The stored fields for documents</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>
+ * <td>.tim</td>
+ * <td>The term dictionary, stores term info</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Term Index}</td>
+ * <td>.tip</td>
+ * <td>The index into the Term Dictionary</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>
+ * <td>.doc</td>
+ * <td>Contains the list of docs which contain each term along with frequency</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Positions}</td>
+ * <td>.pos</td>
+ * <td>Stores position information about where a term occurs in the index</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat Payloads}</td>
+ * <td>.pay</td>
+ * <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Norms}</td>
+ * <td>.nvd, .nvm</td>
+ * <td>Encodes length and boost factors for docs and fields</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-Document Values}</td>
+ * <td>.dvd, .dvm</td>
+ * <td>Encodes additional scoring factors or other per-document information.</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Index}</td>
+ * <td>.tvx</td>
+ * <td>Stores offset into the document data file</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Data}</td>
+ * <td>.tvd</td>
+ * <td>Contains term vector data.</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live Documents}</td>
+ * <td>.liv</td>
+ * <td>Info about what documents are live</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}</td>
+ * <td>.kdd, .kdi, .kdm</td>
+ * <td>Holds indexed points</td>
+ * </tr>
+ * <tr>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}</td>
+ * <td>.vec, .vem, .veq, vex</td>
+ * <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data,
+ * <code>.vem</code> the vector metadata, <code>.veq</code> the quantized vector data, and <code>.vex</code> the
+ * hnsw graph data.</td>
+ * </tr>
+ * </table>
+ *
+ * </div> <a id="Lock_File"></a>
+ *
+ * <h3>Lock File</h3>
+ *
+ * The write lock, which is stored in the index directory by default, is named "write.lock". If the
+ * lock directory is different from the index directory then the write lock will be named
+ * "XXXX-write.lock" where XXXX is a unique prefix derived from the full path to the index
+ * directory. When this file is present, a writer is currently modifying the index (adding or
+ * removing documents). This lock file ensures that only one writer is modifying the index at a
+ * time. <a id="History"></a>
+ *
+ * <h3>History</h3>
+ *
+ * <p>Compatibility notes are provided in this document, describing how file formats have changed
+ * from prior versions:
+ *
+ * <ul>
+ * <li>In version 2.1, the file format was changed to allow lock-less commits (ie, no more commit
+ * lock). The change is fully backwards compatible: you can open a pre-2.1 index for searching
+ * or adding/deleting of docs. When the new segments file is saved (committed), it will be
+ * written in the new file format (meaning no specific "upgrade" process is needed). But note
+ * that once a commit has occurred, pre-2.1 Lucene will not be able to read the index.
+ * <li>In version 2.3, the file format was changed to allow segments to share a single set of doc
+ * store (vectors &amp; stored fields) files. This allows for faster indexing in certain
+ * cases. The change is fully backwards compatible (in the same way as the lock-less commits
+ * change in 2.1).
+ * <li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not Java's modified
+ * UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">LUCENE-510</a> for
+ * details.
+ * <li>In version 2.9, an optional opaque Map&lt;String,String&gt; CommitUserData may be passed to
+ * IndexWriter's commit methods (and later retrieved), which is recorded in the segments_N
+ * file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">LUCENE-1382</a> for
+ * details. Also, diagnostics were added to each segment written recording details about why
+ * it was written (due to flush, merge; which OS/JRE was used; etc.). See issue <a
+ * href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.
+ * <li>In version 3.0, compressed fields are no longer written to the index (they can still be
+ * read, but on merge the new segment will write them, uncompressed). See issue <a
+ * href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a> for details.
+ * <li>In version 3.1, segments records the code version that created them. See <a
+ * href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
+ * Additionally segments track explicitly whether or not they have term vectors. See <a
+ * href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a> for details.
+ * <li>In version 3.2, numeric fields are written as natively to stored fields file, previously
+ * they were stored in text format only.
+ * <li>In version 3.4, fields can omit position data while still indexing term frequencies.
+ * <li>In version 4.0, the format of the inverted index became extensible via the {@link
+ * org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage ({@code DocValues})
+ * was introduced. Normalization factors need no longer be a single byte, they can be any
+ * {@link org.apache.lucene.index.NumericDocValues NumericDocValues}. Terms need not be
+ * unicode strings, they can be any byte sequence. Term offsets can optionally be indexed into
+ * the postings lists. Payloads can be stored in the term vectors.
+ * <li>In version 4.1, the format of the postings list changed to use either of FOR compression or
+ * variable-byte encoding, depending upon the frequency of the term. Terms appearing only once
+ * were changed to inline directly into the term dictionary. Stored fields are compressed by
+ * default.
+ * <li>In version 4.2, term vectors are compressed by default. DocValues has a new multi-valued
+ * type (SortedSet), that can be used for faceting/grouping/joining on multi-valued fields.
+ * <li>In version 4.5, DocValues were extended to explicitly represent missing values.
+ * <li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
+ * allow updating NumericDocValues fields.
+ * <li>In version 4.8, checksum footers were added to the end of each index file for improved data
+ * integrity. Specifically, the last 8 bytes of every index file contain the zlib-crc32
+ * checksum of the file.
+ * <li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric) that is
+ * suitable for faceting/sorting/analytics.
+ * <li>In version 5.4, DocValues have been improved to store more information on disk: addresses
+ * for binary fields and ord indexes for multi-valued fields.
+ * <li>In version 6.0, Points were added, for multi-dimensional range/distance search.
+ * <li>In version 6.2, new Segment info format that reads/writes the index sort, to support index
+ * sorting.
+ * <li>In version 7.0, DocValues have been improved to better support sparse doc values thanks to
+ * an iterator API.
+ * <li>In version 8.0, postings have been enhanced to record, for each block of doc ids, the (term
+ * freq, normalization factor) pairs that may trigger the maximum score of the block. This
+ * information is recorded alongside skip data in order to be able to skip blocks of doc ids
+ * if they may not produce high enough scores. Additionally doc values and norms has been
+ * extended with jump-tables to make access O(1) instead of O(n), where n is the number of
+ * elements to skip when advancing in the data.
+ * <li>In version 8.4, postings, positions, offsets and payload lengths have move to a more
+ * performant encoding that is vectorized.
+ * <li>In version 8.6, index sort serialization is delegated to the sorts themselves, to allow
+ * user-defined sorts to be used
+ * <li>In version 8.6, points fields split the index tree and leaf data into separate files, to
+ * allow for different access patterns to the different data structures
+ * <li>In version 8.7, stored fields compression became adaptive to better handle documents with
+ * smaller stored fields.
+ * <li>In version 9.0, vector-valued fields were added.
+ * <li>In version 9.1, vector-valued fields were modified to add a graph hierarchy.
+ * <li>In version 9.2, docs of vector-valued fields were moved from .vem to .vec and encoded by
+ * IndexDISI. ordToDoc mappings was added to .vem.
+ * <li>In version 9.5, HNSW graph connections were changed to be delta-encoded with vints.
+ * Additionally, metadata file size improvements were made by delta-encoding nodes by graph
+ * layer and not writing the node ids for the zeroth layer.
+ * <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector
+ * format to utilize int8 quantized vectors for float32 vector search.
+ * <li>In version 9.12, skip data was refactored to have only two levels: every 128 docs and every
+ * 4,06 docs, and to be inlined in postings lists. This resulted in a speedup for queries that
+ * need skipping, especially conjunctions.
+ * <li>In version 10.1, block encoding changed to be optimized for int[] storage instead of
+ * long[].
+ * <li>In version 10.3, the index of block tree changed to be specialized trie instead of FST.
+ * </ul>
+ *
+ * <a id="Limitations"></a>
+ *
+ * <h3>Limitations</h3>
+ *
+ * <div>
+ *
+ * <p>Lucene uses a Java <code>int</code> to refer to document numbers, and the index file format
+ * uses an <code>Int32</code> on-disk to store document numbers. This is a limitation of both the
+ * index file format and the current implementation. Eventually these should be replaced with either
+ * <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
+ * VInt} values which have no limit. </div>
+ */
+package org.apache.lucene.backward_codecs.lucene103;
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 9f6cf615a3e..15bb6e9fa56 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -26,3 +26,4 @@ org.apache.lucene.backward_codecs.lucene99.Lucene99Codec
org.apache.lucene.backward_codecs.lucene912.Lucene912Codec
org.apache.lucene.backward_codecs.lucene100.Lucene100Codec
org.apache.lucene.backward_codecs.lucene101.Lucene101Codec
+org.apache.lucene.backward_codecs.lucene103.Lucene103Codec
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 84cf4182384..a0c1266b755 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -19,3 +19,4 @@ org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat
org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat
+org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsWriter.java
similarity index 97%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsWriter.java
index d49e6a0f2e1..7e733340473 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsWriter.java
@@ -14,26 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.LEVEL1_MASK;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.META_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.LEVEL1_MASK;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.META_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.Impact;
import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103RWPostingsFormat.java
new file mode 100644
index 00000000000..1796325cdc1
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/Lucene103RWPostingsFormat.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.IOUtils;
+
+/** Read-write impersonation of {@link Lucene103PostingsFormat}. */
+public final class Lucene103RWPostingsFormat extends Lucene103PostingsFormat {
+
+ private final int version;
+ private final int minTermBlockSize;
+ private final int maxTermBlockSize;
+
+ /** Creates {@code Lucene103PostingsFormat} with default settings. */
+ public Lucene103RWPostingsFormat() {
+ this(
+ Lucene103BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene103BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ }
+
+ /**
+ * Creates {@code Lucene103PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * maxBlockSize} passed to block terms dictionary.
+ *
+ * @see
+ * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
+ */
+ public Lucene103RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
+ }
+
+ /** Expert constructor that allows setting the version. */
+ public Lucene103RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
+ super();
+ if (version < VERSION_START || version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version out of range: " + version);
+ }
+ this.version = version;
+ Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ this.minTermBlockSize = minTermBlockSize;
+ this.maxTermBlockSize = maxTermBlockSize;
+ }
+
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state, version);
+ try {
+ return new Lucene103BlockTreeTermsWriter(
+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, postingsWriter);
+ throw t;
+ }
+ }
+}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestBitSetUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestBitSetUtil.java
new file mode 100644
index 00000000000..094d2111f3a
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestBitSetUtil.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import static org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;
+
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.FixedBitSet;
+
+public class TestBitSetUtil extends LuceneTestCase {
+
+ public void testDenseBitsetToArray() throws Exception {
+ for (int iter = 0; iter < 1000; iter++) {
+ FixedBitSet bitSet = new FixedBitSet(BLOCK_SIZE + random().nextInt(200));
+ while (bitSet.cardinality() < BLOCK_SIZE) {
+ bitSet.set(random().nextInt(bitSet.length()));
+ }
+
+ int[] array = new int[BLOCK_SIZE + 1];
+ int from, to, size;
+ if (random().nextBoolean()) {
+ from = 0;
+ to = bitSet.length();
+ } else {
+ from = random().nextInt(bitSet.length());
+ to = from + random().nextInt(bitSet.length() - from);
+ }
+
+ int base = random().nextInt(1000);
+ size = BitSetUtil.denseBitsetToArray(bitSet, from, to, base, array);
+
+ assertEquals(bitSet.cardinality(from, to), size);
+
+ int[] index = new int[] {0};
+ bitSet.forEach(from, to, base, bit -> assertEquals(bit, array[index[0]++]));
+ assertEquals(size, index[0]);
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForDeltaUtil.java
similarity index 93%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForDeltaUtil.java
index 224731bdaf1..5fa53436adc 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForDeltaUtil.java
@@ -14,12 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -67,8 +66,7 @@ public class TestForDeltaUtil extends LuceneTestCase {
{
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
- PostingDecodingUtil pdu =
- Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);
ForDeltaUtil forDeltaUtil = new ForDeltaUtil();
for (int i = 0; i < iterations; ++i) {
int base = 0;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForUtil.java
new file mode 100644
index 00000000000..8cdada44475
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestForUtil.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+public class TestForUtil extends LuceneTestCase {
+
+ public void testEncodeDecode() throws IOException {
+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
+
+ for (int i = 0; i < iterations; ++i) {
+ final int bpv = TestUtil.nextInt(random(), 1, 31);
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ values[i * ForUtil.BLOCK_SIZE + j] =
+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
+ }
+ }
+
+ final Directory d = new ByteBuffersDirectory();
+ final long endPointer;
+
+ {
+ // encode
+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+ final ForUtil forUtil = new ForUtil();
+
+ for (int i = 0; i < iterations; ++i) {
+ int[] source = new int[ForUtil.BLOCK_SIZE];
+ long or = 0;
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ or |= source[j];
+ }
+ final int bpv = PackedInts.bitsRequired(or);
+ out.writeByte((byte) bpv);
+ forUtil.encode(source, bpv, out);
+ }
+ endPointer = out.getFilePointer();
+ out.close();
+ }
+
+ {
+ // decode
+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);
+ ForUtil forUtil = new ForUtil();
+ for (int i = 0; i < iterations; ++i) {
+ final int bitsPerValue = in.readByte();
+ final long currentFilePointer = in.getFilePointer();
+ final int[] restored = new int[ForUtil.BLOCK_SIZE];
+ forUtil.decode(bitsPerValue, pdu, restored);
+ int[] ints = new int[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ ints[j] = Math.toIntExact(restored[j]);
+ }
+ assertArrayEquals(
+ Arrays.toString(ints),
+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),
+ ints);
+ assertEquals(ForUtil.numBytes(bitsPerValue), in.getFilePointer() - currentFilePointer);
+ }
+ assertEquals(endPointer, in.getFilePointer());
+ in.close();
+ }
+
+ d.close();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestLucene103PostingsFormat.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestLucene103PostingsFormat.java
index 4bceeaae6ee..49538ce371e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestLucene103PostingsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.backward_codecs.lucene103;
import java.io.IOException;
import java.util.ArrayList;
@@ -46,7 +46,7 @@ public class TestLucene103PostingsFormat extends BasePostingsFormatTestCase {
@Override
protected Codec getCodec() {
- return TestUtil.alwaysPostingsFormat(new Lucene103PostingsFormat());
+ return TestUtil.alwaysPostingsFormat(new Lucene103RWPostingsFormat());
}
public void testVInt15() throws IOException {
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPForUtil.java
new file mode 100644
index 00000000000..6d13212bf1a
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPForUtil.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+public class TestPForUtil extends LuceneTestCase {
+
+ public void testEncodeDecode() throws IOException {
+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+ final int[] values = createTestData(iterations, 31);
+
+ final Directory d = new ByteBuffersDirectory();
+ final long endPointer = encodeTestData(iterations, values, d);
+
+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);
+ final PForUtil pforUtil = new PForUtil();
+ for (int i = 0; i < iterations; ++i) {
+ if (random().nextInt(5) == 0) {
+ PForUtil.skip(in);
+ continue;
+ }
+ final int[] restored = new int[ForUtil.BLOCK_SIZE];
+ pforUtil.decode(pdu, restored);
+ int[] ints = new int[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ ints[j] = Math.toIntExact(restored[j]);
+ }
+ assertArrayEquals(
+ Arrays.toString(ints),
+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),
+ ints);
+ }
+ assertEquals(endPointer, in.getFilePointer());
+ in.close();
+
+ d.close();
+ }
+
+ private int[] createTestData(int iterations, int maxBpv) {
+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
+
+ for (int i = 0; i < iterations; ++i) {
+ final int bpv = TestUtil.nextInt(random(), 0, maxBpv);
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ values[i * ForUtil.BLOCK_SIZE + j] =
+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
+ if (random().nextInt(100) == 0) {
+ final int exceptionBpv;
+ if (random().nextInt(10) == 0) {
+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 9, 16), maxBpv);
+ } else {
+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 1, 8), maxBpv);
+ }
+ values[i * ForUtil.BLOCK_SIZE + j] |= random().nextInt(1 << (exceptionBpv - bpv)) << bpv;
+ }
+ }
+ }
+
+ return values;
+ }
+
+ private long encodeTestData(int iterations, int[] values, Directory d) throws IOException {
+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+ final PForUtil pforUtil = new PForUtil();
+
+ for (int i = 0; i < iterations; ++i) {
+ int[] source = new int[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ }
+ pforUtil.encode(source, out);
+ }
+ final long endPointer = out.getFilePointer();
+ out.close();
+
+ return endPointer;
+ }
+}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPostingsUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPostingsUtil.java
new file mode 100644
index 00000000000..cce53a9f87e
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene103/TestPostingsUtil.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene103;
+
+import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestPostingsUtil extends LuceneTestCase {
+
+ // checks for bug described in https://github.com/apache/lucene/issues/13373
+ public void testIntegerOverflow() throws IOException {
+ // Size that writes the first value as a regular vint
+ int randomSize1 = random().nextInt(1, 3);
+ // Size that writes the first value as a group vint
+ int randomSize2 = random().nextInt(4, ForUtil.BLOCK_SIZE);
+ doTestIntegerOverflow(randomSize1);
+ doTestIntegerOverflow(randomSize2);
+ }
+
+ private void doTestIntegerOverflow(int size) throws IOException {
+ final int[] docDeltaBuffer = new int[size];
+ final int[] freqBuffer = new int[size];
+
+ final int delta = 1 << 30;
+ docDeltaBuffer[0] = delta;
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("test", IOContext.DEFAULT)) {
+ // In old implementation, this would cause integer overflow exception.
+ PostingsUtil.writeVIntBlock(out, docDeltaBuffer, freqBuffer, size, true);
+ }
+ int[] restoredDocs = new int[size];
+ int[] restoredFreqs = new int[size];
+ try (IndexInput in = dir.openInput("test", IOContext.DEFAULT)) {
+ PostingsUtil.readVIntBlock(in, restoredDocs, restoredFreqs, size, true, true);
+ }
+ assertEquals(delta, restoredDocs[0]);
+ }
+ }
+}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
index a2c09f53526..384ce889c5a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
@@ -21,9 +21,7 @@ import static org.apache.lucene.backward_index.TestBasicBackwardsCompatibility.a
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import java.io.IOException;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
@@ -69,14 +67,10 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTe
}
protected Codec getCodec() {
- return new Lucene103Codec() {
- @Override
- public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswScalarQuantizedVectorsFormat(
+ return TestUtil.alwaysKnnVectorsFormat(
+ new Lucene99HnswScalarQuantizedVectorsFormat(
Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
- Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
- }
- };
+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH));
}
@Override
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java
index c72809da847..6139b375947 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java
@@ -21,9 +21,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.codecs.lucene103.ForDeltaUtil;
-import org.apache.lucene.codecs.lucene103.ForUtil;
-import org.apache.lucene.codecs.lucene103.PostingIndexInput;
+import org.apache.lucene.codecs.lucene104.ForUtil;
+import org.apache.lucene.codecs.lucene104.PostingIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -60,7 +59,6 @@ public class PostingIndexInputBenchmark {
private IndexInput in;
private PostingIndexInput postingIn;
private final ForUtil forUtil = new ForUtil();
- private final ForDeltaUtil forDeltaUtil = new ForDeltaUtil();
private final int[] values = new int[ForUtil.BLOCK_SIZE];
@Param({"2", "3", "4", "5", "6", "7", "8", "9", "10"})
@@ -78,7 +76,7 @@ public class PostingIndexInputBenchmark {
}
}
in = dir.openInput("docs", IOContext.DEFAULT);
- postingIn = new PostingIndexInput(in, forUtil, forDeltaUtil);
+ postingIn = new PostingIndexInput(in, forUtil);
}
@TearDown(Level.Trial)
@@ -99,13 +97,6 @@ public class PostingIndexInputBenchmark {
bh.consume(values);
}
- @Benchmark
- public void decodeAndPrefixSum(Blackhole bh) throws IOException {
- in.seek(3); // random unaligned offset
- postingIn.decodeAndPrefixSum(bpv, 100, values);
- bh.consume(values);
- }
-
@Benchmark
@Fork(jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})
public void decodeVector(Blackhole bh) throws IOException {
@@ -113,12 +104,4 @@ public class PostingIndexInputBenchmark {
postingIn.decode(bpv, values);
bh.consume(values);
}
-
- @Benchmark
- @Fork(jvmArgsPrepend = {"--add-modules=jdk.incubator.vector"})
- public void decodeAndPrefixSumVector(Blackhole bh) throws IOException {
- in.seek(3); // random unaligned offset
- postingIn.decodeAndPrefixSum(bpv, 100, values);
- bh.consume(values);
- }
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
index f27908bf1e9..36195f66012 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
@@ -22,14 +22,14 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
-/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene103PostingsWriter}. */
+/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene104PostingsWriter}. */
public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
private final int minTermBlockSize;
@@ -67,7 +67,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);
try {
return new OrdsBlockTreeTermsWriter(
@@ -80,7 +80,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);
try {
return new OrdsBlockTreeTermsReader(postingsReader, state);
} catch (Throwable t) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
index 6a14c85430d..cbf6c7e660f 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
@@ -24,7 +24,7 @@ import java.util.TreeMap;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
@@ -54,7 +54,7 @@ import org.apache.lucene.util.automaton.TransitionAccessor;
// - or: longer dense skip lists than just next byte?
/**
- * Wraps {@link Lucene103PostingsFormat} format for on-disk storage, but then at read time loads and
+ * Wraps {@link Lucene104PostingsFormat} format for on-disk storage, but then at read time loads and
* stores all terms and postings directly in RAM as byte[], int[].
*
* <p><b>WARNING</b>: This is exceptionally RAM intensive: it makes no effort to compress the
@@ -97,12 +97,12 @@ public final class DirectPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return PostingsFormat.forName("Lucene103").fieldsConsumer(state);
+ return PostingsFormat.forName("Lucene104").fieldsConsumer(state);
}
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- FieldsProducer postings = PostingsFormat.forName("Lucene103").fieldsProducer(state);
+ FieldsProducer postings = PostingsFormat.forName("Lucene104").fieldsProducer(state);
if (state.context.context() != IOContext.Context.MERGE) {
FieldsProducer loadedPostings;
try {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
index e535ccd33db..a72fde421d6 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
@@ -22,8 +22,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -41,7 +41,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);
try {
return new FSTTermsWriter(state, postingsWriter);
@@ -53,7 +53,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);
try {
return new FSTTermsReader(state, postingsReader);
} catch (Throwable t) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
index a4a9048a73e..af15ab034ee 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
@@ -17,13 +17,13 @@
package org.apache.lucene.codecs.uniformsplit;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.BLOCK_SIZE;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.TermState;
@@ -34,7 +34,7 @@ import org.apache.lucene.util.RamUsageEstimator;
/**
* {@link TermState} serializer which encodes each file pointer as a delta relative to a base file
- * pointer. It differs from {@link Lucene103PostingsWriter#encodeTerm} which encodes each file
+ * pointer. It differs from {@link Lucene104PostingsWriter#encodeTerm} which encodes each file
* pointer as a delta relative to the previous file pointer.
*
* <p>It automatically sets the base file pointer to the first valid file pointer for doc start FP,
@@ -95,7 +95,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Writes a {@link BlockTermState} to the provided {@link DataOutput}.
*
- * <p>Simpler variant of {@link Lucene103PostingsWriter#encodeTerm(DataOutput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene104PostingsWriter#encodeTerm(DataOutput, FieldInfo,
* BlockTermState, boolean)}.
*/
public void writeTermState(
@@ -145,7 +145,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Reads a {@link BlockTermState} from the provided {@link DataInput}.
*
- * <p>Simpler variant of {@link Lucene103PostingsReader#decodeTerm(DataInput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene104PostingsReader#decodeTerm(DataInput, FieldInfo,
* BlockTermState, boolean)}.
*
* @param reuse {@link BlockTermState} to reuse; or null to create a new one.
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
index 64c3a4f5609..801143ae232 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -113,7 +113,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);
try {
return createUniformSplitTermsWriter(
postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
@@ -125,7 +125,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);
try {
return createUniformSplitTermsReader(postingsReader, state, blockDecoder);
} catch (Throwable t) {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
index a4ac5603913..fedeb0ecf52 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
@@ -28,7 +28,7 @@
* org.apache.lucene.search.PhraseQuery})
* <li>Quite efficient for {@link org.apache.lucene.search.PrefixQuery}
* <li>Not efficient for spell-check and {@link org.apache.lucene.search.FuzzyQuery}, in this case
- * prefer {@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat}
+ * prefer {@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat}
* </ul>
*/
package org.apache.lucene.codecs.uniformsplit;
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
index 2f6dc6fa749..66a923dc0b4 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
@@ -17,7 +17,7 @@
package org.apache.lucene.codecs.lucene90.tests;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;
/** Test utility class to create mock {@link IntBlockTermState}. */
public class MockTermStateFactory {
diff --git a/lucene/core/src/generated/checksums/generateForUtil.json b/lucene/core/src/generated/checksums/generateForUtil.json
index 8a50acc81d5..e7970804e68 100644
--- a/lucene/core/src/generated/checksums/generateForUtil.json
+++ b/lucene/core/src/generated/checksums/generateForUtil.json
@@ -1,4 +1,4 @@
{
- "lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java": "1ea2dac2a26be521a70cf74c37fe9134f7b42cb7",
- "lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py": "cd186fe09ceeedb4e2dd3be191d7461e67fa1c2b"
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java": "5dda079c68e6060217f29010618c7fd807583056",
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py": "4692fed62d9f79554647c5423b96b9e60c9f30eb"
}
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index b9030b52a47..17876735f1f 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -32,7 +32,7 @@ module org.apache.lucene.core {
exports org.apache.lucene.codecs.lucene99;
exports org.apache.lucene.codecs.lucene102;
exports org.apache.lucene.codecs.lucene103.blocktree;
- exports org.apache.lucene.codecs.lucene103;
+ exports org.apache.lucene.codecs.lucene104;
exports org.apache.lucene.codecs.perfield;
exports org.apache.lucene.codecs;
exports org.apache.lucene.document;
@@ -79,7 +79,7 @@ module org.apache.lucene.core {
provides org.apache.lucene.analysis.TokenizerFactory with
org.apache.lucene.analysis.standard.StandardTokenizerFactory;
provides org.apache.lucene.codecs.Codec with
- org.apache.lucene.codecs.lucene103.Lucene103Codec;
+ org.apache.lucene.codecs.lucene104.Lucene104Codec;
provides org.apache.lucene.codecs.DocValuesFormat with
org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
@@ -89,7 +89,7 @@ module org.apache.lucene.core {
org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat;
provides org.apache.lucene.codecs.PostingsFormat with
- org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+ org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
provides org.apache.lucene.index.SortFieldProvider with
org.apache.lucene.search.SortField.Provider,
org.apache.lucene.search.SortedNumericSortField.Provider,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
index 026c3b7b36c..b35d7a3f0bb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
@@ -56,7 +56,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
}
@SuppressWarnings("NonFinalStaticField")
- static Codec defaultCodec = LOADER.lookup("Lucene103");
+ static Codec defaultCodec = LOADER.lookup("Lucene104");
}
private final String name;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
index 05fbb64cd83..335a8d24813 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
@@ -151,15 +151,15 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term dictionary}. A
+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term dictionary}. A
* dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Frequency data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Frequency data}. For
* each term in the dictionary, the numbers of all the documents that contain that term, and
* the frequency of the term in that document, unless frequencies are omitted ({@link
* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Proximity data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Proximity data}. For
* each term in the dictionary, the positions that the term occurs in each document. Note that
* this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/BitSetUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/BitSetUtil.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/BitSetUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/BitSetUtil.java
index da1b90fd39f..d5d72d35d84 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/BitSetUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/BitSetUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
import java.util.Objects;
import org.apache.lucene.util.FixedBitSet;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java
new file mode 100644
index 00000000000..3a2ad7396c7
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/ForUtil.java
@@ -0,0 +1,535 @@
+// This file has been automatically generated, DO NOT EDIT
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
+import org.apache.lucene.store.DataOutput;
+
+/**
+ * Inspired from https://fulmicoton.com/posts/bitpacking/ Encodes multiple integers in one to get
+ * SIMD-like speedups. If bitsPerValue &lt;= 8 then we pack 4 ints per Java int else if bitsPerValue
+ * &lt;= 16 we pack 2 ints per Java int else we do scalar operations.
+ */
+public final class ForUtil {
+
+ /** Number of integers per block. */
+ public static final int BLOCK_SIZE = 256;
+
+ static final int BLOCK_SIZE_LOG2 = 8;
+
+ static int expandMask16(int mask16) {
+ return mask16 | (mask16 << 16);
+ }
+
+ static int expandMask8(int mask8) {
+ return expandMask16(mask8 | (mask8 << 8));
+ }
+
+ static int mask32(int bitsPerValue) {
+ return (1 << bitsPerValue) - 1;
+ }
+
+ static int mask16(int bitsPerValue) {
+ return expandMask16((1 << bitsPerValue) - 1);
+ }
+
+ static int mask8(int bitsPerValue) {
+ return expandMask8((1 << bitsPerValue) - 1);
+ }
+
+ static void expand8(int[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ int l = arr[i];
+ arr[i] = (l >>> 24) & 0xFF;
+ arr[64 + i] = (l >>> 16) & 0xFF;
+ arr[128 + i] = (l >>> 8) & 0xFF;
+ arr[192 + i] = l & 0xFF;
+ }
+ }
+
+ static void collapse8(int[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ arr[i] = (arr[i] << 24) | (arr[64 + i] << 16) | (arr[128 + i] << 8) | arr[192 + i];
+ }
+ }
+
+ static void expand16(int[] arr) {
+ for (int i = 0; i < 128; ++i) {
+ int l = arr[i];
+ arr[i] = (l >>> 16) & 0xFFFF;
+ arr[128 + i] = l & 0xFFFF;
+ }
+ }
+
+ static void collapse16(int[] arr) {
+ for (int i = 0; i < 128; ++i) {
+ arr[i] = (arr[i] << 16) | arr[128 + i];
+ }
+ }
+
+ private final int[] tmp = new int[BLOCK_SIZE];
+
+ /** Encode 256 integers from {@code ints} into {@code out}. */
+ void encode(int[] ints, int bitsPerValue, DataOutput out) throws IOException {
+ final int nextPrimitive;
+ if (bitsPerValue <= 8) {
+ nextPrimitive = 8;
+ collapse8(ints);
+ } else if (bitsPerValue <= 16) {
+ nextPrimitive = 16;
+ collapse16(ints);
+ } else {
+ nextPrimitive = 32;
+ }
+ encode(ints, bitsPerValue, nextPrimitive, out, tmp);
+ }
+
+ static void encode(int[] ints, int bitsPerValue, int primitiveSize, DataOutput out, int[] tmp)
+ throws IOException {
+ final int numInts = BLOCK_SIZE * primitiveSize / Integer.SIZE;
+
+ final int numIntsPerShift = bitsPerValue * 8;
+ int idx = 0;
+ int shift = primitiveSize - bitsPerValue;
+ for (int i = 0; i < numIntsPerShift; ++i) {
+ tmp[i] = ints[idx++] << shift;
+ }
+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
+ for (int i = 0; i < numIntsPerShift; ++i) {
+ tmp[i] |= ints[idx++] << shift;
+ }
+ }
+
+ final int remainingBitsPerInt = shift + bitsPerValue;
+ final int maskRemainingBitsPerInt;
+ if (primitiveSize == 8) {
+ maskRemainingBitsPerInt = MASKS8[remainingBitsPerInt];
+ } else if (primitiveSize == 16) {
+ maskRemainingBitsPerInt = MASKS16[remainingBitsPerInt];
+ } else {
+ maskRemainingBitsPerInt = MASKS32[remainingBitsPerInt];
+ }
+
+ int tmpIdx = 0;
+ int remainingBitsPerValue = bitsPerValue;
+ while (idx < numInts) {
+ if (remainingBitsPerValue >= remainingBitsPerInt) {
+ remainingBitsPerValue -= remainingBitsPerInt;
+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerInt;
+ if (remainingBitsPerValue == 0) {
+ idx++;
+ remainingBitsPerValue = bitsPerValue;
+ }
+ } else {
+ final int mask1, mask2;
+ if (primitiveSize == 8) {
+ mask1 = MASKS8[remainingBitsPerValue];
+ mask2 = MASKS8[remainingBitsPerInt - remainingBitsPerValue];
+ } else if (primitiveSize == 16) {
+ mask1 = MASKS16[remainingBitsPerValue];
+ mask2 = MASKS16[remainingBitsPerInt - remainingBitsPerValue];
+ } else {
+ mask1 = MASKS32[remainingBitsPerValue];
+ mask2 = MASKS32[remainingBitsPerInt - remainingBitsPerValue];
+ }
+ tmp[tmpIdx] |= (ints[idx++] & mask1) << (remainingBitsPerInt - remainingBitsPerValue);
+ remainingBitsPerValue = bitsPerValue - remainingBitsPerInt + remainingBitsPerValue;
+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & mask2;
+ }
+ }
+
+ for (int i = 0; i < numIntsPerShift; ++i) {
+ out.writeInt(tmp[i]);
+ }
+ }
+
+ /** Number of bytes required to encode 256 integers of {@code bitsPerValue} bits per value. */
+ static int numBytes(int bitsPerValue) {
+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
+ }
+
+ static void decodeSlow(int bitsPerValue, PostingDecodingUtil pdu, int[] tmp, int[] ints)
+ throws IOException {
+ final int numInts = bitsPerValue << 3;
+ final int mask = MASKS32[bitsPerValue];
+ pdu.splitInts(numInts, ints, 32 - bitsPerValue, 32, mask, tmp, 0, -1);
+ final int remainingBitsPerInt = 32 - bitsPerValue;
+ final int mask32RemainingBitsPerInt = MASKS32[remainingBitsPerInt];
+ int tmpIdx = 0;
+ int remainingBits = remainingBitsPerInt;
+ for (int intsIdx = numInts; intsIdx < BLOCK_SIZE; ++intsIdx) {
+ int b = bitsPerValue - remainingBits;
+ int l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
+ while (b >= remainingBitsPerInt) {
+ b -= remainingBitsPerInt;
+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerInt) << b;
+ }
+ if (b > 0) {
+ l |= (tmp[tmpIdx] >>> (remainingBitsPerInt - b)) & MASKS32[b];
+ remainingBits = remainingBitsPerInt - b;
+ } else {
+ remainingBits = remainingBitsPerInt;
+ }
+ ints[intsIdx] = l;
+ }
+ }
+
+ static final int[] MASKS8 = new int[8];
+ static final int[] MASKS16 = new int[16];
+ static final int[] MASKS32 = new int[32];
+
+ static {
+ for (int i = 0; i < 8; ++i) {
+ MASKS8[i] = mask8(i);
+ }
+ for (int i = 0; i < 16; ++i) {
+ MASKS16[i] = mask16(i);
+ }
+ for (int i = 0; i < 32; ++i) {
+ MASKS32[i] = mask32(i);
+ }
+ }
+
+ // mark values in array as final ints to avoid the cost of reading array, arrays should only be
+ // used when the idx is a variable
+ static final int MASK8_1 = MASKS8[1];
+ static final int MASK8_2 = MASKS8[2];
+ static final int MASK8_3 = MASKS8[3];
+ static final int MASK8_4 = MASKS8[4];
+ static final int MASK8_5 = MASKS8[5];
+ static final int MASK8_6 = MASKS8[6];
+ static final int MASK8_7 = MASKS8[7];
+ static final int MASK16_1 = MASKS16[1];
+ static final int MASK16_2 = MASKS16[2];
+ static final int MASK16_3 = MASKS16[3];
+ static final int MASK16_4 = MASKS16[4];
+ static final int MASK16_5 = MASKS16[5];
+ static final int MASK16_6 = MASKS16[6];
+ static final int MASK16_7 = MASKS16[7];
+ static final int MASK16_8 = MASKS16[8];
+ static final int MASK16_9 = MASKS16[9];
+ static final int MASK16_10 = MASKS16[10];
+ static final int MASK16_11 = MASKS16[11];
+ static final int MASK16_12 = MASKS16[12];
+ static final int MASK16_13 = MASKS16[13];
+ static final int MASK16_14 = MASKS16[14];
+ static final int MASK16_15 = MASKS16[15];
+ static final int MASK32_1 = MASKS32[1];
+ static final int MASK32_2 = MASKS32[2];
+ static final int MASK32_3 = MASKS32[3];
+ static final int MASK32_4 = MASKS32[4];
+ static final int MASK32_5 = MASKS32[5];
+ static final int MASK32_6 = MASKS32[6];
+ static final int MASK32_7 = MASKS32[7];
+ static final int MASK32_8 = MASKS32[8];
+ static final int MASK32_9 = MASKS32[9];
+ static final int MASK32_10 = MASKS32[10];
+ static final int MASK32_11 = MASKS32[11];
+ static final int MASK32_12 = MASKS32[12];
+ static final int MASK32_13 = MASKS32[13];
+ static final int MASK32_14 = MASKS32[14];
+ static final int MASK32_15 = MASKS32[15];
+ static final int MASK32_16 = MASKS32[16];
+
+ /** Decode 256 integers into {@code ints}. */
+ void decode(int bitsPerValue, PostingDecodingUtil pdu, int[] ints) throws IOException {
+ switch (bitsPerValue) {
+ case 1:
+ decode1(pdu, ints);
+ expand8(ints);
+ break;
+ case 2:
+ decode2(pdu, ints);
+ expand8(ints);
+ break;
+ case 3:
+ decode3(pdu, tmp, ints);
+ expand8(ints);
+ break;
+ case 4:
+ decode4(pdu, ints);
+ expand8(ints);
+ break;
+ case 5:
+ decode5(pdu, tmp, ints);
+ expand8(ints);
+ break;
+ case 6:
+ decode6(pdu, tmp, ints);
+ expand8(ints);
+ break;
+ case 7:
+ decode7(pdu, tmp, ints);
+ expand8(ints);
+ break;
+ case 8:
+ decode8(pdu, ints);
+ expand8(ints);
+ break;
+ case 9:
+ decode9(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 10:
+ decode10(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 11:
+ decode11(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 12:
+ decode12(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 13:
+ decode13(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 14:
+ decode14(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 15:
+ decode15(pdu, tmp, ints);
+ expand16(ints);
+ break;
+ case 16:
+ decode16(pdu, ints);
+ expand16(ints);
+ break;
+ default:
+ decodeSlow(bitsPerValue, pdu, tmp, ints);
+ break;
+ }
+ }
+
+ static void decode1(PostingDecodingUtil pdu, int[] ints) throws IOException {
+ pdu.splitInts(8, ints, 7, 1, MASK8_1, ints, 56, MASK8_1);
+ }
+
+ static void decode2(PostingDecodingUtil pdu, int[] ints) throws IOException {
+ pdu.splitInts(16, ints, 6, 2, MASK8_2, ints, 48, MASK8_2);
+ }
+
+ static void decode3(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(24, ints, 5, 3, MASK8_3, tmp, 0, MASK8_2);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 48; iter < 8; ++iter, tmpIdx += 3, intsIdx += 2) {
+ int l0 = tmp[tmpIdx + 0] << 1;
+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;
+ ints[intsIdx + 0] = l0;
+ int l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;
+ l1 |= tmp[tmpIdx + 2] << 0;
+ ints[intsIdx + 1] = l1;
+ }
+ }
+
+ static void decode4(PostingDecodingUtil pdu, int[] ints) throws IOException {
+ pdu.splitInts(32, ints, 4, 4, MASK8_4, ints, 32, MASK8_4);
+ }
+
+ static void decode5(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(40, ints, 3, 5, MASK8_5, tmp, 0, MASK8_3);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 40; iter < 8; ++iter, tmpIdx += 5, intsIdx += 3) {
+ int l0 = tmp[tmpIdx + 0] << 2;
+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;
+ ints[intsIdx + 0] = l0;
+ int l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;
+ l1 |= tmp[tmpIdx + 2] << 1;
+ l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;
+ ints[intsIdx + 1] = l1;
+ int l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;
+ l2 |= tmp[tmpIdx + 4] << 0;
+ ints[intsIdx + 2] = l2;
+ }
+ }
+
+ static void decode6(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(48, ints, 2, 6, MASK8_6, tmp, 0, MASK8_2);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 48; iter < 16; ++iter, tmpIdx += 3, intsIdx += 1) {
+ int l0 = tmp[tmpIdx + 0] << 4;
+ l0 |= tmp[tmpIdx + 1] << 2;
+ l0 |= tmp[tmpIdx + 2] << 0;
+ ints[intsIdx + 0] = l0;
+ }
+ }
+
+ static void decode7(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(56, ints, 1, 7, MASK8_7, tmp, 0, MASK8_1);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 56; iter < 8; ++iter, tmpIdx += 7, intsIdx += 1) {
+ int l0 = tmp[tmpIdx + 0] << 6;
+ l0 |= tmp[tmpIdx + 1] << 5;
+ l0 |= tmp[tmpIdx + 2] << 4;
+ l0 |= tmp[tmpIdx + 3] << 3;
+ l0 |= tmp[tmpIdx + 4] << 2;
+ l0 |= tmp[tmpIdx + 5] << 1;
+ l0 |= tmp[tmpIdx + 6] << 0;
+ ints[intsIdx + 0] = l0;
+ }
+ }
+
+ static void decode8(PostingDecodingUtil pdu, int[] ints) throws IOException {
+ pdu.in.readInts(ints, 0, 64);
+ }
+
+ static void decode9(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(72, ints, 7, 9, MASK16_9, tmp, 0, MASK16_7);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 72; iter < 8; ++iter, tmpIdx += 9, intsIdx += 7) {
+ int l0 = tmp[tmpIdx + 0] << 2;
+ l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;
+ ints[intsIdx + 0] = l0;
+ int l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;
+ l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;
+ ints[intsIdx + 1] = l1;
+ int l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;
+ l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;
+ ints[intsIdx + 2] = l2;
+ int l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;
+ l3 |= tmp[tmpIdx + 4] << 1;
+ l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;
+ ints[intsIdx + 3] = l3;
+ int l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;
+ l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;
+ ints[intsIdx + 4] = l4;
+ int l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;
+ l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;
+ ints[intsIdx + 5] = l5;
+ int l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;
+ l6 |= tmp[tmpIdx + 8] << 0;
+ ints[intsIdx + 6] = l6;
+ }
+ }
+
+ static void decode10(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(80, ints, 6, 10, MASK16_10, tmp, 0, MASK16_6);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 80; iter < 16; ++iter, tmpIdx += 5, intsIdx += 3) {
+ int l0 = tmp[tmpIdx + 0] << 4;
+ l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;
+ ints[intsIdx + 0] = l0;
+ int l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;
+ l1 |= tmp[tmpIdx + 2] << 2;
+ l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;
+ ints[intsIdx + 1] = l1;
+ int l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;
+ l2 |= tmp[tmpIdx + 4] << 0;
+ ints[intsIdx + 2] = l2;
+ }
+ }
+
+ static void decode11(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(88, ints, 5, 11, MASK16_11, tmp, 0, MASK16_5);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 88; iter < 8; ++iter, tmpIdx += 11, intsIdx += 5) {
+ int l0 = tmp[tmpIdx + 0] << 6;
+ l0 |= tmp[tmpIdx + 1] << 1;
+ l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;
+ ints[intsIdx + 0] = l0;
+ int l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;
+ l1 |= tmp[tmpIdx + 3] << 2;
+ l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;
+ ints[intsIdx + 1] = l1;
+ int l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;
+ l2 |= tmp[tmpIdx + 5] << 3;
+ l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;
+ ints[intsIdx + 2] = l2;
+ int l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;
+ l3 |= tmp[tmpIdx + 7] << 4;
+ l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;
+ ints[intsIdx + 3] = l3;
+ int l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;
+ l4 |= tmp[tmpIdx + 9] << 5;
+ l4 |= tmp[tmpIdx + 10] << 0;
+ ints[intsIdx + 4] = l4;
+ }
+ }
+
+ static void decode12(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(96, ints, 4, 12, MASK16_12, tmp, 0, MASK16_4);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 96; iter < 32; ++iter, tmpIdx += 3, intsIdx += 1) {
+ int l0 = tmp[tmpIdx + 0] << 8;
+ l0 |= tmp[tmpIdx + 1] << 4;
+ l0 |= tmp[tmpIdx + 2] << 0;
+ ints[intsIdx + 0] = l0;
+ }
+ }
+
+ static void decode13(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(104, ints, 3, 13, MASK16_13, tmp, 0, MASK16_3);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 104; iter < 8; ++iter, tmpIdx += 13, intsIdx += 3) {
+ int l0 = tmp[tmpIdx + 0] << 10;
+ l0 |= tmp[tmpIdx + 1] << 7;
+ l0 |= tmp[tmpIdx + 2] << 4;
+ l0 |= tmp[tmpIdx + 3] << 1;
+ l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;
+ ints[intsIdx + 0] = l0;
+ int l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;
+ l1 |= tmp[tmpIdx + 5] << 8;
+ l1 |= tmp[tmpIdx + 6] << 5;
+ l1 |= tmp[tmpIdx + 7] << 2;
+ l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;
+ ints[intsIdx + 1] = l1;
+ int l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;
+ l2 |= tmp[tmpIdx + 9] << 9;
+ l2 |= tmp[tmpIdx + 10] << 6;
+ l2 |= tmp[tmpIdx + 11] << 3;
+ l2 |= tmp[tmpIdx + 12] << 0;
+ ints[intsIdx + 2] = l2;
+ }
+ }
+
+ static void decode14(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(112, ints, 2, 14, MASK16_14, tmp, 0, MASK16_2);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 112; iter < 16; ++iter, tmpIdx += 7, intsIdx += 1) {
+ int l0 = tmp[tmpIdx + 0] << 12;
+ l0 |= tmp[tmpIdx + 1] << 10;
+ l0 |= tmp[tmpIdx + 2] << 8;
+ l0 |= tmp[tmpIdx + 3] << 6;
+ l0 |= tmp[tmpIdx + 4] << 4;
+ l0 |= tmp[tmpIdx + 5] << 2;
+ l0 |= tmp[tmpIdx + 6] << 0;
+ ints[intsIdx + 0] = l0;
+ }
+ }
+
+ static void decode15(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {
+ pdu.splitInts(120, ints, 1, 15, MASK16_15, tmp, 0, MASK16_1);
+ for (int iter = 0, tmpIdx = 0, intsIdx = 120; iter < 8; ++iter, tmpIdx += 15, intsIdx += 1) {
+ int l0 = tmp[tmpIdx + 0] << 14;
+ l0 |= tmp[tmpIdx + 1] << 13;
+ l0 |= tmp[tmpIdx + 2] << 12;
+ l0 |= tmp[tmpIdx + 3] << 11;
+ l0 |= tmp[tmpIdx + 4] << 10;
+ l0 |= tmp[tmpIdx + 5] << 9;
+ l0 |= tmp[tmpIdx + 6] << 8;
+ l0 |= tmp[tmpIdx + 7] << 7;
+ l0 |= tmp[tmpIdx + 8] << 6;
+ l0 |= tmp[tmpIdx + 9] << 5;
+ l0 |= tmp[tmpIdx + 10] << 4;
+ l0 |= tmp[tmpIdx + 11] << 3;
+ l0 |= tmp[tmpIdx + 12] << 2;
+ l0 |= tmp[tmpIdx + 13] << 1;
+ l0 |= tmp[tmpIdx + 14] << 0;
+ ints[intsIdx + 0] = l0;
+ }
+ }
+
+ static void decode16(PostingDecodingUtil pdu, int[] ints) throws IOException {
+ pdu.in.readInts(ints, 0, 128);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104Codec.java
new file mode 100644
index 00000000000..f2b7317bc93
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104Codec.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.util.Objects;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
+import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat;
+import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
+import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+
+/**
+ * Implements the Lucene 10.4 index format
+ *
+ * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
+ *
+ * @see org.apache.lucene.codecs.lucene104 package documentation for file format details.
+ * @lucene.experimental
+ */
+public class Lucene104Codec extends Codec {
+
+ /** Configuration option for the codec. */
+ public enum Mode {
+ /** Trade compression ratio for retrieval speed. */
+ BEST_SPEED(Lucene90StoredFieldsFormat.Mode.BEST_SPEED),
+ /** Trade retrieval speed for compression ratio. */
+ BEST_COMPRESSION(Lucene90StoredFieldsFormat.Mode.BEST_COMPRESSION);
+
+ private final Lucene90StoredFieldsFormat.Mode storedMode;
+
+ private Mode(Lucene90StoredFieldsFormat.Mode storedMode) {
+ this.storedMode = Objects.requireNonNull(storedMode);
+ }
+ }
+
+ private final TermVectorsFormat vectorsFormat = new Lucene90TermVectorsFormat();
+ private final FieldInfosFormat fieldInfosFormat = new Lucene94FieldInfosFormat();
+ private final SegmentInfoFormat segmentInfosFormat = new Lucene99SegmentInfoFormat();
+ private final LiveDocsFormat liveDocsFormat = new Lucene90LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene90CompoundFormat();
+ private final NormsFormat normsFormat = new Lucene90NormsFormat();
+
+ private final PostingsFormat defaultPostingsFormat;
+ private final PostingsFormat postingsFormat =
+ new PerFieldPostingsFormat() {
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return Lucene104Codec.this.getPostingsFormatForField(field);
+ }
+ };
+
+ private final DocValuesFormat defaultDVFormat;
+ private final DocValuesFormat docValuesFormat =
+ new PerFieldDocValuesFormat() {
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return Lucene104Codec.this.getDocValuesFormatForField(field);
+ }
+ };
+
+ private final KnnVectorsFormat defaultKnnVectorsFormat;
+ private final KnnVectorsFormat knnVectorsFormat =
+ new PerFieldKnnVectorsFormat() {
+ @Override
+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+ return Lucene104Codec.this.getKnnVectorsFormatForField(field);
+ }
+ };
+
+ private final StoredFieldsFormat storedFieldsFormat;
+
+ /** Instantiates a new codec. */
+ public Lucene104Codec() {
+ this(Mode.BEST_SPEED);
+ }
+
+ /**
+ * Instantiates a new codec, specifying the stored fields compression mode to use.
+ *
+ * @param mode stored fields compression mode to use for newly flushed/merged segments.
+ */
+ public Lucene104Codec(Mode mode) {
+ super("Lucene104");
+ this.storedFieldsFormat =
+ new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);
+ this.defaultPostingsFormat = new Lucene104PostingsFormat();
+ this.defaultDVFormat = new Lucene90DocValuesFormat();
+ this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
+ }
+
+ @Override
+ public final StoredFieldsFormat storedFieldsFormat() {
+ return storedFieldsFormat;
+ }
+
+ @Override
+ public final TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
+
+ @Override
+ public final PostingsFormat postingsFormat() {
+ return postingsFormat;
+ }
+
+ @Override
+ public final FieldInfosFormat fieldInfosFormat() {
+ return fieldInfosFormat;
+ }
+
+ @Override
+ public final SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfosFormat;
+ }
+
+ @Override
+ public final LiveDocsFormat liveDocsFormat() {
+ return liveDocsFormat;
+ }
+
+ @Override
+ public final CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
+
+ @Override
+ public final PointsFormat pointsFormat() {
+ return new Lucene90PointsFormat();
+ }
+
+ @Override
+ public final KnnVectorsFormat knnVectorsFormat() {
+ return knnVectorsFormat;
+ }
+
+ /**
+ * Returns the postings format that should be used for writing new segments of <code>field</code>.
+ *
+ * <p>The default implementation always returns "Lucene104".
+ *
+ * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+ * future version of Lucene are only guaranteed to be able to read the default implementation,
+ */
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return defaultPostingsFormat;
+ }
+
+ /**
+ * Returns the docvalues format that should be used for writing new segments of <code>field</code>
+ * .
+ *
+ * <p>The default implementation always returns "Lucene90".
+ *
+ * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+ * future version of Lucene are only guaranteed to be able to read the default implementation.
+ */
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return defaultDVFormat;
+ }
+
+ /**
+ * Returns the vectors format that should be used for writing new segments of <code>field</code>
+ *
+ * <p>The default implementation always returns "Lucene99HnswVectorsFormat".
+ *
+ * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
+ * future version of Lucene are only guaranteed to be able to read the default implementation.
+ */
+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+ return defaultKnnVectorsFormat;
+ }
+
+ @Override
+ public final DocValuesFormat docValuesFormat() {
+ return docValuesFormat;
+ }
+
+ @Override
+ public final NormsFormat normsFormat() {
+ return normsFormat;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsFormat.java
new file mode 100644
index 00000000000..b4828145aba
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsFormat.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PostingsReaderBase;
+import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+/**
+ * Lucene 10.4 postings format, which encodes postings in packed integer blocks for fast decode.
+ *
+ * <p>Basic idea:
+ *
+ * <ul>
+ * <li><b>Packed Blocks and VInt Blocks</b>:
+ * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed
+ * format}): the block size (i.e. number of integers inside block) is fixed (currently 256).
+ * Additionally blocks that are all the same value are encoded in an optimized way.
+ * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block
+ * size is variable.
+ * <li><b>Block structure</b>:
+ * <p>When the postings are long enough, Lucene104PostingsFormat will try to encode most
+ * integer data as a packed block.
+ * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as
+ * two packed blocks, while the remaining 3 are encoded as one VInt block.
+ * <p>Different kinds of data are always encoded separately into different packed blocks, but
+ * may possibly be interleaved into the same VInt block.
+ * <p>This strategy is applied to pairs: &lt;document number, frequency&gt;, &lt;position,
+ * payload length&gt;, &lt;position, offset start, offset length&gt;, and &lt;position,
+ * payload length, offsetstart, offset length&gt;.
+ * <li><b>Skipdata</b>:
+ * <p>Skipdata is interleaved with blocks on 2 levels. Level 0 skip data is interleaved
+ * between every packed block. Level 1 skip data is interleaved between every 32 packed
+ * blocks.
+ * <li><b>Positions, Payloads, and Offsets</b>:
+ * <p>A position is an integer indicating where the term occurs within one document. A payload
+ * is a blob of metadata associated with current position. An offset is a pair of integers
+ * indicating the tokenized start/end offsets for given term in current position: it is
+ * essentially a specialized payload.
+ * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets
+ * (assuming a null payload contributes one count). As mentioned in block structure, it is
+ * possible to encode these three either combined or separately.
+ * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,
+ * position data is separated out as .pos, while payloads and offsets are encoded in .pay
+ * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all
+ * these three are stored interleaved into the .pos (so is payload metadata).
+ * <p>With this strategy, the majority of payload and offset data will be outside .pos file.
+ * So for queries that require only position data, running on a full index with payloads and
+ * offsets, this reduces disk pre-fetches.
+ * </ul>
+ *
+ * <p>Files and detailed format:
+ *
+ * <ul>
+ * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>
+ * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>
+ * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>
+ * <li><code>.pos</code>: <a href="#Positions">Positions</a>
+ * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>
+ * </ul>
+ *
+ * <a id="Termdictionary"></a>
+ *
+ * <dl>
+ * <dd><b>Term Dictionary</b>
+ * <p>The .tim file contains the list of terms in each field along with per-term statistics
+ * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
+ * .doc, .pos, and .pay files. See {@link Lucene103BlockTreeTermsWriter} for more details on
+ * the format.
+ * <p>NOTE: The term dictionary can plug into different postings implementations: the postings
+ * writer/reader are actually responsible for encoding and decoding the PostingsHeader and
+ * TermMetadata sections described here:
+ * <ul>
+ * <li>PostingsHeader --&gt; Header, PackedBlockSize
+ * <li>TermMetadata --&gt; (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,
+ * PayFPDelta?
+ * <li>Header, --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>PackedBlockSize, SingletonDocID --&gt; {@link DataOutput#writeVInt VInt}
+ * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta --&gt; {@link
+ * DataOutput#writeVLong VLong}
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version
+ * information for the postings.
+ * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width
+ * is determined by the largest integer. Smaller block size result in smaller variance
+ * among width of integers hence smaller indexes. Larger block size result in more
+ * efficient bulk i/o hence better acceleration. This value should always be a multiple
+ * of 64, currently fixed as 256 as a tradeoff. It is also the skip interval used to
+ * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.
+ * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In
+ * particular, it is the difference of file offset between this term's data and previous
+ * term's data (or zero, for the first term in the block).On disk it is stored as the
+ * difference from previous value in sequence.
+ * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.
+ * While PayFPDelta determines the position of this term's &lt;TermPayloads,
+ * TermOffsets?&gt; within the .pay file. Similar to DocFPDelta, it is the difference
+ * between two file positions (or neglected, for fields that omit payloads and offsets).
+ * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last
+ * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or
+ * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to
+ * load following payloads and offsets from .pos instead of .pay. Every time a new block
+ * of positions are to be loaded, the PostingsReader will use this value to check
+ * whether current block is packed format or VInt. When packed format, payloads and
+ * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when
+ * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).
+ * <li>SingletonDocID is an optimization when a term only appears in one document. In this
+ * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a
+ * VIntBlock at that location, the single document ID is written to the term dictionary.
+ * </ul>
+ * </dl>
+ *
+ * <a id="Termindex"></a>
+ *
+ * <dl>
+ * <dd><b>Term Index</b>
+ * <p>The .tip file contains an index into the term dictionary, so that it can be accessed
+ * randomly. See {@link Lucene103BlockTreeTermsWriter} for more details on the format.
+ * </dl>
+ *
+ * <a id="Frequencies"></a>
+ *
+ * <dl>
+ * <dd><b>Frequencies and Skip Data</b>
+ * <p>The .doc file contains the lists of documents which contain each term, along with the
+ * frequency of the term in that document (except when frequencies are omitted: {@link
+ * IndexOptions#DOCS}). Skip data is saved at the end of each term's postings. The skip data
+ * is saved once for the entire postings list.
+ * <ul>
+ * <li>docFile(.doc) --&gt; Header, &lt;TermFreqs&gt;<sup>TermCount</sup>, Footer
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>TermFreqs --&gt; &lt;PackedBlock32&gt; <sup>PackedDocBlockNum/32</sup>, VIntBlock?
+ * <li>PackedBlock32 --&gt; Level1SkipData, &lt;PackedBlock&gt; <sup>32</sup>
+ * <li>PackedBlock --&gt; Level0SkipData, PackedDocDeltaBlock, PackedFreqBlock?
+ * <li>VIntBlock --&gt;
+ * &lt;DocDelta[,Freq?]&gt;<sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
+ * <li>Level1SkipData --&gt; DocDelta, DocFPDelta, Skip1NumBytes?, ImpactLength?, Impacts?,
+ * PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
+ * <li>Level0SkipData --&gt; Skip0NumBytes, DocDelta, DocFPDelta, PackedBlockLength,
+ * ImpactLength?, Impacts?, PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
+ * <li>PackedFreqBlock --&gt; {@link PackedInts PackedInts}, uses patching
+ * <li>PackedDocDeltaBlock --&gt; {@link PackedInts PackedInts}, does not use patching
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>PackedDocDeltaBlock is theoretically generated from two steps:
+ * <ol>
+ * <li>Calculate the difference between each document number and previous one, and get
+ * a d-gaps list (for the first document, use absolute value);
+ * <li>For those d-gaps from first one to
+ * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed
+ * blocks.
+ * </ol>
+ * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.
+ * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a
+ * format that encodes DocDelta and Freq:
+ * <p>DocDelta: if frequencies are indexed, this determines both the document number and
+ * the frequency. In particular, DocDelta/2 is the difference between this document
+ * number and the previous document number (or zero when this is the first document in a
+ * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the
+ * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the
+ * gap (not multiplied by 2) between document numbers and no frequency information is
+ * stored.
+ * <p>For example, the TermFreqs for a term which occurs once in document seven and
+ * three times in document eleven, with frequencies indexed, would be the following
+ * sequence of VInts:
+ * <p>15, 8, 3
+ * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence
+ * of VInts instead:
+ * <p>7,4
+ * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or
+ * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)
+ * <li>On skip data, DocDelta is the delta between the last doc of the previous block - or
+ * -1 if there is no previous block - and the last doc of this block. This helps know by
+ * how much the doc ID should be incremented in case the block gets skipped.
+ * <li>Skip0Length is the length of skip data at level 0. Encoding it is useful when skip
+ * data is never needed to quickly skip over skip data, e.g. if only using nextDoc(). It
+ * is also used when only the first fields of skip data are needed, in order to skip
+ * over remaining fields without reading them.
+ * <li>ImpactLength and Impacts are only stored if frequencies are indexed.
+ * <li>Since positions and payloads are also block encoded, the skip should skip to related
+ * block first, then fetch the values according to in-block offset. PosFPSkip and
+ * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.
+ * While PosBlockOffset indicates which value to fetch inside the related block
+ * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as
+ * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,
+ * and stored as a difference sequence.
+ * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to
+ * the sum of the payload lengths in the current block up to PosBlockOffset
+ * <li>ImpactLength is the total length of CompetitiveFreqDelta and CompetitiveNormDelta
+ * pairs. CompetitiveFreqDelta and CompetitiveNormDelta are used to safely skip score
+ * calculation for uncompetitive documents; See {@link
+ * org.apache.lucene.codecs.CompetitiveImpactAccumulator} for more details.
+ * </ul>
+ * </dl>
+ *
+ * <a id="Positions"></a>
+ *
+ * <dl>
+ * <dd><b>Positions</b>
+ * <p>The .pos file contains the lists of positions that each term occurs at within documents.
+ * It also sometimes stores part of payloads and offsets for speedup.
+ * <ul>
+ * <li>PosFile(.pos) --&gt; Header, &lt;TermPositions&gt; <sup>TermCount</sup>, Footer
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>TermPositions --&gt; &lt;PackedPosDeltaBlock&gt; <sup>PackedPosBlockNum</sup>,
+ * VIntBlock?
+ * <li>VIntBlock --&gt; &lt;PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,
+ * OffsetLength?&gt;<sup>PosVIntCount</sup>
+ * <li>PackedPosDeltaBlock --&gt; {@link PackedInts PackedInts}
+ * <li>PositionDelta, OffsetDelta, OffsetLength --&gt; {@link DataOutput#writeVInt VInt}
+ * <li>PayloadData --&gt; {@link DataOutput#writeByte byte}<sup>PayLength</sup>
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and
+ * position values for each term document pair are incremental, and ordered by document
+ * number.
+ * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,
+ * payloads or offsets. In particular, PackedPosBlockNum =
+ * floor(totalTermFreq/PackedBlockSize)
+ * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,
+ * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize
+ * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock
+ * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.
+ * <li>PositionDelta is, if payloads are disabled for the term's field, the difference
+ * between the position of the current occurrence in the document and the previous
+ * occurrence (or zero, if this is the first occurrence in this document). If payloads
+ * are enabled for the term's field, then PositionDelta/2 is the difference between the
+ * current and the previous position. If payloads are enabled and PositionDelta is odd,
+ * then PayloadLength is stored, indicating the length of the payload at the current
+ * term position.
+ * <li>For example, the TermPositions for a term which occurs as the fourth term in one
+ * document, and as the fifth and ninth term in a subsequent document, would be the
+ * following sequence of VInts (payloads disabled):
+ * <p>4, 5, 4
+ * <li>PayloadData is metadata associated with the current term position. If PayloadLength
+ * is stored at the current position, then it indicates the length of this payload. If
+ * PayloadLength is not stored, then this payload has the same length as the payload at
+ * the previous position.
+ * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous
+ * occurrence (or zero, if this is the first occurrence in this document). If
+ * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous
+ * occurrence and an OffsetLength follows. Offset data is only written for {@link
+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.
+ * </ul>
+ * </dl>
+ *
+ * <a id="Payloads"></a>
+ *
+ * <dl>
+ * <dd><b>Payloads and Offsets</b>
+ * <p>The .pay file will store payloads and offsets associated with certain term-document
+ * positions. Some payloads and offsets will be separated out into .pos file, for performance
+ * reasons.
+ * <ul>
+ * <li>PayFile(.pay): --&gt; Header, &lt;TermPayloads?, TermOffsets?&gt;
+ * <sup>TermCount</sup>, Footer
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>TermPayloads --&gt; &lt;PackedPayLengthBlock, SumPayLength, PayData&gt;
+ * <sup>PackedPayBlockNum</sup>
+ * <li>TermOffsets --&gt; &lt;PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock&gt;
+ * <sup>PackedPayBlockNum</sup>
+ * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock --&gt;
+ * {@link PackedInts PackedInts}
+ * <li>SumPayLength --&gt; {@link DataOutput#writeVInt VInt}
+ * <li>PayData --&gt; {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that
+ * part of payload/offsets are stored in .pos.
+ * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is
+ * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip
+ * Data</a>. While PackedStartDeltaBlock follows a same procedure as
+ * PackedDocDeltaBlock.
+ * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also
+ * synonym for PackedOffsetBlockNum.
+ * <li>SumPayLength is the total length of payloads written within one block, should be the
+ * sum of PayLengths in one packed block.
+ * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the
+ * current position.
+ * </ul>
+ * </dl>
+ *
+ * @lucene.experimental
+ */
+public final class Lucene104PostingsFormat extends PostingsFormat {
+
+ /** Filename extension for some small metadata about how postings are encoded. */
+ public static final String META_EXTENSION = "psm";
+
+ /**
+ * Filename extension for document number, frequencies, and skip data. See chapter: <a
+ * href="#Frequencies">Frequencies and Skip Data</a>
+ */
+ public static final String DOC_EXTENSION = "doc";
+
+ /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */
+ public static final String POS_EXTENSION = "pos";
+
+ /**
+ * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and
+ * Offsets</a>
+ */
+ public static final String PAY_EXTENSION = "pay";
+
+ /** Size of blocks. */
+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
+
+ static final int BLOCK_MASK = BLOCK_SIZE - 1;
+
+ /** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */
+ public static final int LEVEL1_FACTOR = 32;
+
+ /** Total number of docs covered by level 1 skip data: 32 * 256 = 8,192 */
+ public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;
+
+ static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
+
+ /**
+ * Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is
+ * internally used to help the JVM make good inlining decisions.
+ *
+ * @lucene.internal
+ */
+ public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {
+ return Lucene104PostingsReader.BlockPostingsEnum.class;
+ }
+
+ static final String TERMS_CODEC = "Lucene104PostingsWriterTerms";
+ static final String META_CODEC = "Lucene104PostingsWriterMeta";
+ static final String DOC_CODEC = "Lucene104PostingsWriterDoc";
+ static final String POS_CODEC = "Lucene104PostingsWriterPos";
+ static final String PAY_CODEC = "Lucene104PostingsWriterPay";
+
+ static final int VERSION_START = 0;
+
+ static final int VERSION_CURRENT = VERSION_START;
+
+ private final int version;
+ private final int minTermBlockSize;
+ private final int maxTermBlockSize;
+
+ /** Creates {@code Lucene104PostingsFormat} with default settings. */
+ public Lucene104PostingsFormat() {
+ this(
+ Lucene103BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene103BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ }
+
+ /**
+ * Creates {@code Lucene104PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * maxBlockSize} passed to block terms dictionary.
+ *
+ * @see
+ * Lucene103BlockTreeTermsWriter#Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
+ */
+ public Lucene104PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
+ }
+
+ /** Expert constructor that allows setting the version. */
+ public Lucene104PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
+ super("Lucene104");
+ if (version < VERSION_START || version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version out of range: " + version);
+ }
+ this.version = version;
+ Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ this.minTermBlockSize = minTermBlockSize;
+ this.maxTermBlockSize = maxTermBlockSize;
+ }
+
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state, version);
+ try {
+ return new Lucene103BlockTreeTermsWriter(
+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, postingsWriter);
+ throw t;
+ }
+ }
+
+ @Override
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);
+ try {
+ return new Lucene103BlockTreeTermsReader(postingsReader, state);
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, postingsReader);
+ throw t;
+ }
+ }
+
+ /**
+ * Holds all state required for {@link Lucene104PostingsReader} to produce a {@link
+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
+ *
+ * @lucene.internal
+ */
+ public static final class IntBlockTermState extends BlockTermState {
+ /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */
+ public long docStartFP;
+
+ /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */
+ public long posStartFP;
+
+ /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */
+ public long payStartFP;
+
+ /**
+ * file offset for the last position in the last block, if there are more than {@link
+ * ForUtil#BLOCK_SIZE} positions; otherwise -1
+ *
+ * <p>One might think to use total term frequency to track how many positions are left to read
+ * as we decode the blocks, and decode the last block differently when num_left_positions &lt;
+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip
+ * blocks as the skipper will only tell us new position offset (start of block) and number of
+ * positions to skip for that block, without telling us how many positions it has skipped.
+ */
+ public long lastPosBlockOffset;
+
+ /**
+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
+ * totalTermFreq in this case.
+ */
+ public int singletonDocID;
+
+ /** Sole constructor. */
+ public IntBlockTermState() {
+ lastPosBlockOffset = -1;
+ singletonDocID = -1;
+ }
+
+ @Override
+ public IntBlockTermState clone() {
+ IntBlockTermState other = new IntBlockTermState();
+ other.copyFrom(this);
+ return other;
+ }
+
+ @Override
+ public void copyFrom(TermState _other) {
+ super.copyFrom(_other);
+ IntBlockTermState other = (IntBlockTermState) _other;
+ docStartFP = other.docStartFP;
+ posStartFP = other.posStartFP;
+ payStartFP = other.payStartFP;
+ lastPosBlockOffset = other.lastPosBlockOffset;
+ singletonDocID = other.singletonDocID;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString()
+ + " docStartFP="
+ + docStartFP
+ + " posStartFP="
+ + posStartFP
+ + " payStartFP="
+ + payStartFP
+ + " lastPosBlockOffset="
+ + lastPosBlockOffset
+ + " singletonDocID="
+ + singletonDocID;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsReader.java
new file mode 100644
index 00000000000..97d75989a70
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsReader.java
@@ -0,0 +1,1498 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import static org.apache.lucene.codecs.lucene104.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.LEVEL1_NUM_DOCS;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.META_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.VERSION_START;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.PostingsReaderBase;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FreqAndNormBuffer;
+import org.apache.lucene.index.Impacts;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
+import org.apache.lucene.internal.vectorization.VectorizationProvider;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.VectorUtil;
+
+/**
+ * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.
+ *
+ * @lucene.experimental
+ */
+public final class Lucene104PostingsReader extends PostingsReaderBase {
+
+ static final VectorizationProvider VECTORIZATION_PROVIDER = VectorizationProvider.getInstance();
+
+ private final IndexInput docIn;
+ private final IndexInput posIn;
+ private final IndexInput payIn;
+
+ private final int maxNumImpactsAtLevel0;
+ private final int maxImpactNumBytesAtLevel0;
+ private final int maxNumImpactsAtLevel1;
+ private final int maxImpactNumBytesAtLevel1;
+
+ /** Sole constructor. */
+ public Lucene104PostingsReader(SegmentReadState state) throws IOException {
+ String metaName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.META_EXTENSION);
+ final long expectedDocFileLength, expectedPosFileLength, expectedPayFileLength;
+ int version;
+ try (ChecksumIndexInput metaIn = state.directory.openChecksumInput(metaName)) {
+ try {
+ version =
+ CodecUtil.checkIndexHeader(
+ metaIn,
+ META_CODEC,
+ VERSION_START,
+ VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ maxNumImpactsAtLevel0 = metaIn.readInt();
+ maxImpactNumBytesAtLevel0 = metaIn.readInt();
+ maxNumImpactsAtLevel1 = metaIn.readInt();
+ maxImpactNumBytesAtLevel1 = metaIn.readInt();
+ expectedDocFileLength = metaIn.readLong();
+ if (state.fieldInfos.hasProx()) {
+ expectedPosFileLength = metaIn.readLong();
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ expectedPayFileLength = metaIn.readLong();
+ } else {
+ expectedPayFileLength = -1;
+ }
+ } else {
+ expectedPosFileLength = -1;
+ expectedPayFileLength = -1;
+ }
+ CodecUtil.checkFooter(metaIn, null);
+ } catch (Throwable t) {
+ if (metaIn != null) {
+ CodecUtil.checkFooter(metaIn, t);
+ throw new AssertionError("unreachable");
+ } else {
+ throw t;
+ }
+ }
+ }
+
+ IndexInput docIn = null;
+ IndexInput posIn = null;
+ IndexInput payIn = null;
+
+ // NOTE: these data files are too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+
+ String docName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.DOC_EXTENSION);
+ try {
+ docIn =
+ state.directory.openInput(
+ docName, state.context.withHints(FileTypeHint.DATA, FileDataHint.POSTINGS));
+ CodecUtil.checkIndexHeader(
+ docIn, DOC_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(docIn, expectedDocFileLength);
+
+ if (state.fieldInfos.hasProx()) {
+ String proxName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.POS_EXTENSION);
+ posIn = state.directory.openInput(proxName, state.context.withHints(FileTypeHint.DATA));
+ CodecUtil.checkIndexHeader(
+ posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(posIn, expectedPosFileLength);
+
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ String payName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name,
+ state.segmentSuffix,
+ Lucene104PostingsFormat.PAY_EXTENSION);
+ payIn = state.directory.openInput(payName, state.context.withHints(FileTypeHint.DATA));
+ CodecUtil.checkIndexHeader(
+ payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(payIn, expectedPayFileLength);
+ }
+ }
+
+ this.docIn = docIn;
+ this.posIn = posIn;
+ this.payIn = payIn;
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, docIn, posIn, payIn);
+ throw t;
+ }
+ }
+
+ @Override
+ public void init(IndexInput termsIn, SegmentReadState state) throws IOException {
+ // Make sure we are talking to the matching postings writer
+ CodecUtil.checkIndexHeader(
+ termsIn,
+ TERMS_CODEC,
+ VERSION_START,
+ VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ final int indexBlockSize = termsIn.readVInt();
+ if (indexBlockSize != BLOCK_SIZE) {
+ throw new IllegalStateException(
+ "index-time BLOCK_SIZE ("
+ + indexBlockSize
+ + ") != read-time BLOCK_SIZE ("
+ + BLOCK_SIZE
+ + ")");
+ }
+ }
+
+ static void prefixSum(int[] buffer, int count, long base) {
+ buffer[0] += base;
+ for (int i = 1; i < count; ++i) {
+ buffer[i] += buffer[i - 1];
+ }
+ }
+
+ @Override
+ public BlockTermState newTermState() {
+ return new IntBlockTermState();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(docIn, posIn, payIn);
+ }
+
+ @Override
+ public void decodeTerm(
+ DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
+ throws IOException {
+ final IntBlockTermState termState = (IntBlockTermState) _termState;
+ if (absolute) {
+ termState.docStartFP = 0;
+ termState.posStartFP = 0;
+ termState.payStartFP = 0;
+ }
+
+ final long l = in.readVLong();
+ if ((l & 0x01) == 0) {
+ termState.docStartFP += l >>> 1;
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
+ } else {
+ termState.singletonDocID = -1;
+ }
+ } else {
+ assert absolute == false;
+ assert termState.singletonDocID != -1;
+ termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);
+ }
+
+ if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
+ termState.posStartFP += in.readVLong();
+ if (fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0
+ || fieldInfo.hasPayloads()) {
+ termState.payStartFP += in.readVLong();
+ }
+ if (termState.totalTermFreq > BLOCK_SIZE) {
+ termState.lastPosBlockOffset = in.readVLong();
+ } else {
+ termState.lastPosBlockOffset = -1;
+ }
+ }
+ }
+
+ @Override
+ public PostingsEnum postings(
+ FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)
+ throws IOException {
+ return (reuse instanceof BlockPostingsEnum everythingEnum
+ && everythingEnum.canReuse(docIn, fieldInfo, flags, false)
+ ? everythingEnum
+ : new BlockPostingsEnum(fieldInfo, flags, false))
+ .reset((IntBlockTermState) termState, flags);
+ }
+
+ @Override
+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)
+ throws IOException {
+ return new BlockPostingsEnum(fieldInfo, flags, true).reset((IntBlockTermState) state, flags);
+ }
+
+ private static int sumOverRange(int[] arr, int start, int end) {
+ int res = 0;
+ for (int i = start; i < end; i++) {
+ res += arr[i];
+ }
+ return res;
+ }
+
+ final class BlockPostingsEnum extends ImpactsEnum {
+
+ private enum DeltaEncoding {
+ /**
+ * Deltas between consecutive docs are stored as packed integers, ie. the block is encoded
+ * using Frame Of Reference (FOR).
+ */
+ PACKED,
+ /**
+ * Deltas between consecutive docs are stored using unary coding, ie. {@code delta-1} zero
+ * bits followed by a one bit, ie. the block is encoded as an offset plus a bit set.
+ */
+ UNARY
+ }
+
+ private ForUtil forUtil;
+ private PForUtil pforUtil;
+
+ /* Variables that store the content of a block and the current position within this block */
+ /* Shared variables */
+ private DeltaEncoding encoding;
+ private int doc; // doc we last read
+
+ /* Variables when the block is stored as packed deltas (Frame Of Reference) */
+ private final int[] docBuffer = new int[BLOCK_SIZE];
+
+ /* Variables when the block is stored as a bit set */
+ // Since we use a bit set when it's more storage-efficient, the bit set cannot have more than
+ // BLOCK_SIZE*32 bits, which is the maximum possible storage requirement with FOR.
+ private final FixedBitSet docBitSet = new FixedBitSet(BLOCK_SIZE * Integer.SIZE);
+ private int docBitSetBase;
+ // Reuse docBuffer for cumulative pop counts of the words of the bit set.
+ private final int[] docCumulativeWordPopCounts = docBuffer;
+
+ // level 0 skip data
+ private int level0LastDocID;
+ private long level0DocEndFP;
+
+ // level 1 skip data
+ private int level1LastDocID;
+ private long level1DocEndFP;
+ private int level1DocCountUpto;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)
+
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
+
+ private int docCountLeft; // number of remaining docs in this postings list
+ private int prevDocID; // last doc ID of the previous block
+
+ private int docBufferSize;
+ private int docBufferUpto;
+
+ private IndexInput docIn;
+ private PostingDecodingUtil docInUtil;
+
+ private final int[] freqBuffer = new int[BLOCK_SIZE];
+ private final int[] posDeltaBuffer;
+
+ private final int[] payloadLengthBuffer;
+ private final int[] offsetStartDeltaBuffer;
+ private final int[] offsetLengthBuffer;
+
+ private byte[] payloadBytes;
+ private int payloadByteUpto;
+ private int payloadLength;
+
+ private int lastStartOffset;
+ private int startOffset;
+ private int endOffset;
+
+ private int posBufferUpto;
+
+ final IndexInput posIn;
+ final PostingDecodingUtil posInUtil;
+ final IndexInput payIn;
+ final PostingDecodingUtil payInUtil;
+ final BytesRef payload;
+
+ final IndexOptions options;
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+ final boolean indexHasOffsetsOrPayloads;
+
+ final int flags;
+ final boolean needsFreq;
+ final boolean needsPos;
+ final boolean needsOffsets;
+ final boolean needsPayloads;
+ final boolean needsOffsetsOrPayloads;
+ final boolean needsImpacts;
+ final boolean needsDocsAndFreqsOnly;
+
+ private long freqFP; // offset of the freq block
+
+ private int position; // current position
+
+ // value of docBufferUpto on the last doc ID when positions have been read
+ private int posDocBufferUpto;
+
+ // how many positions "behind" we are; nextPosition must
+ // skip these to "catch up":
+ private int posPendingCount;
+
+ // File pointer where the last (vInt encoded) pos delta
+ // block is. We need this to know whether to bulk
+ // decode vs vInt decode the block:
+ private long lastPosBlockFP;
+
+ // level 0 skip data
+ private long level0PosEndFP;
+ private int level0BlockPosUpto;
+ private long level0PayEndFP;
+ private int level0BlockPayUpto;
+ private final BytesRef level0SerializedImpacts;
+
+ // level 1 skip data
+ private long level1PosEndFP;
+ private int level1BlockPosUpto;
+ private long level1PayEndFP;
+ private int level1BlockPayUpto;
+ private final BytesRef level1SerializedImpacts;
+
+ private final FreqAndNormBuffer impactBuffer;
+
+ // true if we shallow-advanced to a new block that we have not decoded yet
+ private boolean needsRefilling;
+
+ public BlockPostingsEnum(FieldInfo fieldInfo, int flags, boolean needsImpacts)
+ throws IOException {
+ options = fieldInfo.getIndexOptions();
+ indexHasFreq = options.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos = options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsets =
+ options.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+ indexHasOffsetsOrPayloads = indexHasOffsets || indexHasPayloads;
+
+ this.flags = flags;
+ needsFreq = indexHasFreq && PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);
+ needsPos = indexHasPos && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);
+ needsOffsets = indexHasOffsets && PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
+ needsPayloads =
+ indexHasPayloads && PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
+ needsOffsetsOrPayloads = needsOffsets || needsPayloads;
+ this.needsImpacts = needsImpacts;
+ needsDocsAndFreqsOnly = needsPos == false && needsImpacts == false;
+
+ if (needsFreq == false) {
+ Arrays.fill(freqBuffer, 1);
+ }
+
+ if (needsImpacts) {
+ impactBuffer = new FreqAndNormBuffer();
+ int capacity = 1; // for dummy impacts
+ if (needsFreq) {
+ capacity = Math.max(maxNumImpactsAtLevel0, capacity);
+ capacity = Math.max(maxNumImpactsAtLevel1, capacity);
+ }
+ impactBuffer.growNoCopy(capacity);
+ } else {
+ impactBuffer = null;
+ }
+
+ if (needsFreq && needsImpacts) {
+ level0SerializedImpacts = new BytesRef(maxImpactNumBytesAtLevel0);
+ level1SerializedImpacts = new BytesRef(maxImpactNumBytesAtLevel1);
+ } else {
+ level0SerializedImpacts = null;
+ level1SerializedImpacts = null;
+ }
+
+ if (needsPos) {
+ this.posIn = Lucene104PostingsReader.this.posIn.clone();
+ posInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(posIn);
+ posDeltaBuffer = new int[BLOCK_SIZE];
+ } else {
+ this.posIn = null;
+ this.posInUtil = null;
+ posDeltaBuffer = null;
+ }
+
+ if (needsOffsets || needsPayloads) {
+ this.payIn = Lucene104PostingsReader.this.payIn.clone();
+ payInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(payIn);
+ } else {
+ this.payIn = null;
+ payInUtil = null;
+ }
+
+ if (needsOffsets) {
+ offsetStartDeltaBuffer = new int[BLOCK_SIZE];
+ offsetLengthBuffer = new int[BLOCK_SIZE];
+ } else {
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ startOffset = -1;
+ endOffset = -1;
+ }
+
+ if (indexHasPayloads) {
+ payloadLengthBuffer = new int[BLOCK_SIZE];
+ payloadBytes = new byte[128];
+ payload = new BytesRef();
+ } else {
+ payloadLengthBuffer = null;
+ payloadBytes = null;
+ payload = null;
+ }
+ }
+
+ public boolean canReuse(
+ IndexInput docIn, FieldInfo fieldInfo, int flags, boolean needsImpacts) {
+ return docIn == Lucene104PostingsReader.this.docIn
+ && options == fieldInfo.getIndexOptions()
+ && indexHasPayloads == fieldInfo.hasPayloads()
+ && this.flags == flags
+ && this.needsImpacts == needsImpacts;
+ }
+
+ public BlockPostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {
+ docFreq = termState.docFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docFreq > 1) {
+ if (docIn == null) {
+ // lazy init
+ docIn = Lucene104PostingsReader.this.docIn.clone();
+ docInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(docIn);
+ }
+ prefetchPostings(docIn, termState);
+ }
+
+ if (forUtil == null && docFreq >= BLOCK_SIZE) {
+ forUtil = new ForUtil();
+ }
+ totalTermFreq = indexHasFreq ? termState.totalTermFreq : termState.docFreq;
+ if (needsFreq && pforUtil == null && totalTermFreq >= BLOCK_SIZE) {
+ if (forUtil == null) {
+ forUtil = new ForUtil();
+ }
+ pforUtil = new PForUtil(forUtil);
+ }
+
+ // Where this term's postings start in the .pos file:
+ final long posTermStartFP = termState.posStartFP;
+ // Where this term's payloads/offsets start in the .pay
+ // file:
+ final long payTermStartFP = termState.payStartFP;
+ if (posIn != null) {
+ posIn.seek(posTermStartFP);
+ if (payIn != null) {
+ payIn.seek(payTermStartFP);
+ }
+ }
+ level1PosEndFP = posTermStartFP;
+ level1PayEndFP = payTermStartFP;
+ level0PosEndFP = posTermStartFP;
+ level0PayEndFP = payTermStartFP;
+ posPendingCount = 0;
+ payloadByteUpto = 0;
+ if (termState.totalTermFreq < BLOCK_SIZE) {
+ lastPosBlockFP = posTermStartFP;
+ } else if (termState.totalTermFreq == BLOCK_SIZE) {
+ lastPosBlockFP = -1;
+ } else {
+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
+ }
+
+ level1BlockPosUpto = 0;
+ level1BlockPayUpto = 0;
+ level0BlockPosUpto = 0;
+ level0BlockPayUpto = 0;
+ posBufferUpto = BLOCK_SIZE;
+
+ doc = -1;
+ prevDocID = -1;
+ docCountLeft = docFreq;
+ freqFP = -1L;
+ level0LastDocID = -1;
+ if (docFreq < LEVEL1_NUM_DOCS) {
+ level1LastDocID = NO_MORE_DOCS;
+ if (docFreq > 1) {
+ docIn.seek(termState.docStartFP);
+ }
+ } else {
+ level1LastDocID = -1;
+ level1DocEndFP = termState.docStartFP;
+ }
+ level1DocCountUpto = 0;
+ docBufferSize = BLOCK_SIZE;
+ docBufferUpto = BLOCK_SIZE;
+ posDocBufferUpto = BLOCK_SIZE;
+
+ return this;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int freq() throws IOException {
+ if (freqFP != -1) {
+ docIn.seek(freqFP);
+ pforUtil.decode(docInUtil, freqBuffer);
+ freqFP = -1;
+ }
+ return freqBuffer[docBufferUpto - 1];
+ }
+
+ private void refillFullBlock() throws IOException {
+ int bitsPerValue = docIn.readByte();
+ if (bitsPerValue > 0) {
+ // block is encoded as 256 packed integers that record the delta between doc IDs
+ forUtil.decode(bitsPerValue, docInUtil, docBuffer);
+ int sum = prevDocID;
+ for (int i = 0; i < BLOCK_SIZE; ++i) {
+ sum += docBuffer[i];
+ docBuffer[i] = sum;
+ }
+ encoding = DeltaEncoding.PACKED;
+ } else {
+ // block is encoded as a bit set
+ assert level0LastDocID != NO_MORE_DOCS;
+ docBitSetBase = prevDocID + 1;
+ int numLongs;
+ if (bitsPerValue == 0) {
+ // 0 is used to record that all 256 docs in the block are consecutive
+ numLongs = BLOCK_SIZE / Long.SIZE; // 2
+ docBitSet.set(0, BLOCK_SIZE);
+ } else {
+ numLongs = -bitsPerValue;
+ docIn.readLongs(docBitSet.getBits(), 0, numLongs);
+ }
+ if (needsFreq) {
+ // Note: we know that BLOCK_SIZE bits are set, so no need to compute the cumulative pop
+ // count at the last index, it will be BLOCK_SIZE.
+ // Note: this for loop auto-vectorizes
+ for (int i = 0; i < numLongs - 1; ++i) {
+ docCumulativeWordPopCounts[i] = Long.bitCount(docBitSet.getBits()[i]);
+ }
+ for (int i = 1; i < numLongs - 1; ++i) {
+ docCumulativeWordPopCounts[i] += docCumulativeWordPopCounts[i - 1];
+ }
+ docCumulativeWordPopCounts[numLongs - 1] = BLOCK_SIZE;
+ assert docCumulativeWordPopCounts[numLongs - 2]
+ + Long.bitCount(docBitSet.getBits()[numLongs - 1])
+ == BLOCK_SIZE;
+ }
+ encoding = DeltaEncoding.UNARY;
+ }
+ if (indexHasFreq) {
+ if (needsFreq) {
+ freqFP = docIn.getFilePointer();
+ }
+ PForUtil.skip(docIn);
+ }
+ docCountLeft -= BLOCK_SIZE;
+ prevDocID = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ posDocBufferUpto = 0;
+ }
+
+ private void refillRemainder() throws IOException {
+ assert docCountLeft >= 0 && docCountLeft < BLOCK_SIZE;
+ if (docFreq == 1) {
+ docBuffer[0] = singletonDocID;
+ freqBuffer[0] = (int) totalTermFreq;
+ docBuffer[1] = NO_MORE_DOCS;
+ assert freqFP == -1;
+ docCountLeft = 0;
+ docBufferSize = 1;
+ } else {
+ // Read vInts:
+ PostingsUtil.readVIntBlock(
+ docIn, docBuffer, freqBuffer, docCountLeft, indexHasFreq, needsFreq);
+ prefixSum(docBuffer, docCountLeft, prevDocID);
+ docBuffer[docCountLeft] = NO_MORE_DOCS;
+ freqFP = -1L;
+ docBufferSize = docCountLeft;
+ docCountLeft = 0;
+ }
+ prevDocID = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ posDocBufferUpto = 0;
+ encoding = DeltaEncoding.PACKED;
+ assert docBuffer[docBufferSize] == NO_MORE_DOCS;
+ }
+
+ private void refillDocs() throws IOException {
+ assert docCountLeft >= 0;
+
+ if (docCountLeft >= BLOCK_SIZE) {
+ refillFullBlock();
+ } else {
+ refillRemainder();
+ }
+ }
+
+ private void skipLevel1To(int target) throws IOException {
+ while (true) {
+ prevDocID = level1LastDocID;
+ level0LastDocID = level1LastDocID;
+ docIn.seek(level1DocEndFP);
+ level0PosEndFP = level1PosEndFP;
+ level0BlockPosUpto = level1BlockPosUpto;
+ level0PayEndFP = level1PayEndFP;
+ level0BlockPayUpto = level1BlockPayUpto;
+ docCountLeft = docFreq - level1DocCountUpto;
+ level1DocCountUpto += LEVEL1_NUM_DOCS;
+
+ if (docCountLeft < LEVEL1_NUM_DOCS) {
+ level1LastDocID = NO_MORE_DOCS;
+ break;
+ }
+
+ level1LastDocID += docIn.readVInt();
+ long delta = docIn.readVLong();
+ level1DocEndFP = delta + docIn.getFilePointer();
+
+ if (indexHasFreq) {
+ long skip1EndFP = docIn.readShort() + docIn.getFilePointer();
+ int numImpactBytes = docIn.readShort();
+ if (needsImpacts && level1LastDocID >= target) {
+ docIn.readBytes(level1SerializedImpacts.bytes, 0, numImpactBytes);
+ level1SerializedImpacts.length = numImpactBytes;
+ } else {
+ docIn.skipBytes(numImpactBytes);
+ }
+ if (indexHasPos) {
+ level1PosEndFP += docIn.readVLong();
+ level1BlockPosUpto = docIn.readByte() & 0xFF;
+ if (indexHasOffsetsOrPayloads) {
+ level1PayEndFP += docIn.readVLong();
+ level1BlockPayUpto = docIn.readVInt();
+ }
+ }
+ assert docIn.getFilePointer() == skip1EndFP;
+ }
+
+ if (level1LastDocID >= target) {
+ break;
+ }
+ }
+ }
+
+ private void doMoveToNextLevel0Block() throws IOException {
+ assert doc == level0LastDocID;
+ if (posIn != null) {
+ if (level0PosEndFP >= posIn.getFilePointer()) {
+ posIn.seek(level0PosEndFP);
+ posPendingCount = level0BlockPosUpto;
+ if (payIn != null) {
+ assert level0PayEndFP >= payIn.getFilePointer();
+ payIn.seek(level0PayEndFP);
+ payloadByteUpto = level0BlockPayUpto;
+ }
+ posBufferUpto = BLOCK_SIZE;
+ } else {
+ assert freqFP == -1L;
+ posPendingCount += sumOverRange(freqBuffer, posDocBufferUpto, BLOCK_SIZE);
+ }
+ }
+
+ if (docCountLeft >= BLOCK_SIZE) {
+ docIn.readVLong(); // level0NumBytes
+ int docDelta = readVInt15(docIn);
+ level0LastDocID += docDelta;
+ long blockLength = readVLong15(docIn);
+ level0DocEndFP = docIn.getFilePointer() + blockLength;
+ if (indexHasFreq) {
+ int numImpactBytes = docIn.readVInt();
+ if (needsImpacts) {
+ docIn.readBytes(level0SerializedImpacts.bytes, 0, numImpactBytes);
+ level0SerializedImpacts.length = numImpactBytes;
+ } else {
+ docIn.skipBytes(numImpactBytes);
+ }
+
+ if (indexHasPos) {
+ level0PosEndFP += docIn.readVLong();
+ level0BlockPosUpto = docIn.readByte() & 0xFF;
+ if (indexHasOffsetsOrPayloads) {
+ level0PayEndFP += docIn.readVLong();
+ level0BlockPayUpto = docIn.readVInt();
+ }
+ }
+ }
+ refillFullBlock();
+ } else {
+ level0LastDocID = NO_MORE_DOCS;
+ refillRemainder();
+ }
+ }
+
+ private void moveToNextLevel0Block() throws IOException {
+ if (doc == level1LastDocID) { // advance level 1 skip data
+ skipLevel1To(doc + 1);
+ }
+
+ // Now advance level 0 skip data
+ prevDocID = level0LastDocID;
+
+ if (needsDocsAndFreqsOnly && docCountLeft >= BLOCK_SIZE) {
+ // Optimize the common path for exhaustive evaluation
+ long level0NumBytes = docIn.readVLong();
+ long level0End = docIn.getFilePointer() + level0NumBytes;
+ level0LastDocID += readVInt15(docIn);
+ docIn.seek(level0End);
+ refillFullBlock();
+ } else {
+ doMoveToNextLevel0Block();
+ }
+ }
+
+ private void readLevel0PosData() throws IOException {
+ level0PosEndFP += docIn.readVLong();
+ level0BlockPosUpto = docIn.readByte() & 0xFF;
+ if (indexHasOffsetsOrPayloads) {
+ level0PayEndFP += docIn.readVLong();
+ level0BlockPayUpto = docIn.readVInt();
+ }
+ }
+
+ private void seekPosData(long posFP, int posUpto, long payFP, int payUpto) throws IOException {
+ // If nextBlockPosFP is less than the current FP, it means that the block of positions for
+ // the first docs of the next block are already decoded. In this case we just accumulate
+ // frequencies into posPendingCount instead of seeking backwards and decoding the same pos
+ // block again.
+ if (posFP >= posIn.getFilePointer()) {
+ posIn.seek(posFP);
+ posPendingCount = posUpto;
+ if (payIn != null) { // needs payloads or offsets
+ assert level0PayEndFP >= payIn.getFilePointer();
+ payIn.seek(payFP);
+ payloadByteUpto = payUpto;
+ }
+ posBufferUpto = BLOCK_SIZE;
+ } else {
+ posPendingCount += sumOverRange(freqBuffer, posDocBufferUpto, BLOCK_SIZE);
+ }
+ }
+
+ private void skipLevel0To(int target) throws IOException {
+ long posFP;
+ int posUpto;
+ long payFP;
+ int payUpto;
+
+ while (true) {
+ prevDocID = level0LastDocID;
+
+ posFP = level0PosEndFP;
+ posUpto = level0BlockPosUpto;
+ payFP = level0PayEndFP;
+ payUpto = level0BlockPayUpto;
+
+ if (docCountLeft >= BLOCK_SIZE) {
+ long numSkipBytes = docIn.readVLong();
+ long skip0End = docIn.getFilePointer() + numSkipBytes;
+ int docDelta = readVInt15(docIn);
+ level0LastDocID += docDelta;
+ boolean found = target <= level0LastDocID;
+ long blockLength = readVLong15(docIn);
+ level0DocEndFP = docIn.getFilePointer() + blockLength;
+
+ if (indexHasFreq) {
+ if (found == false && needsPos == false) {
+ docIn.seek(skip0End);
+ } else {
+ int numImpactBytes = docIn.readVInt();
+ if (needsImpacts && found) {
+ docIn.readBytes(level0SerializedImpacts.bytes, 0, numImpactBytes);
+ level0SerializedImpacts.length = numImpactBytes;
+ } else {
+ docIn.skipBytes(numImpactBytes);
+ }
+
+ if (needsPos) {
+ readLevel0PosData();
+ } else {
+ docIn.seek(skip0End);
+ }
+ }
+ }
+
+ if (found) {
+ break;
+ }
+
+ docIn.seek(level0DocEndFP);
+ docCountLeft -= BLOCK_SIZE;
+ } else {
+ level0LastDocID = NO_MORE_DOCS;
+ break;
+ }
+ }
+
+ if (posIn != null) { // needs positions
+ seekPosData(posFP, posUpto, payFP, payUpto);
+ }
+ }
+
+ @Override
+ public void advanceShallow(int target) throws IOException {
+ if (target > level0LastDocID) { // advance level 0 skip data
+ doAdvanceShallow(target);
+ needsRefilling = true;
+ }
+ }
+
+ private void doAdvanceShallow(int target) throws IOException {
+ if (target > level1LastDocID) { // advance skip data on level 1
+ skipLevel1To(target);
+ } else if (needsRefilling) {
+ docIn.seek(level0DocEndFP);
+ docCountLeft -= BLOCK_SIZE;
+ }
+
+ skipLevel0To(target);
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (doc == level0LastDocID || needsRefilling) {
+ if (needsRefilling) {
+ refillDocs();
+ needsRefilling = false;
+ } else {
+ moveToNextLevel0Block();
+ }
+ }
+
+ switch (encoding) {
+ case PACKED:
+ doc = docBuffer[docBufferUpto];
+ break;
+ case UNARY:
+ int next = docBitSet.nextSetBit(doc - docBitSetBase + 1);
+ assert next != NO_MORE_DOCS;
+ doc = docBitSetBase + next;
+ break;
+ }
+
+ ++docBufferUpto;
+ return this.doc;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > level0LastDocID || needsRefilling) {
+ if (target > level0LastDocID) {
+ doAdvanceShallow(target);
+ }
+ refillDocs();
+ needsRefilling = false;
+ }
+
+ switch (encoding) {
+ case PACKED:
+ {
+ int next = VectorUtil.findNextGEQ(docBuffer, target, docBufferUpto, docBufferSize);
+ this.doc = docBuffer[next];
+ docBufferUpto = next + 1;
+ }
+ break;
+ case UNARY:
+ {
+ int next = docBitSet.nextSetBit(target - docBitSetBase);
+ assert next != NO_MORE_DOCS;
+ this.doc = docBitSetBase + next;
+ if (needsFreq) {
+ int wordIndex = next >> 6;
+ // Take the cumulative pop count for the given word, and subtract bits on the left of
+ // the current doc.
+ docBufferUpto =
+ 1
+ + docCumulativeWordPopCounts[wordIndex]
+ - Long.bitCount(docBitSet.getBits()[wordIndex] >>> next);
+ } else {
+ // When only docs needed and block is UNARY encoded, we do not need to maintain
+ // docBufferUpTo to record the iteration position in the block.
+ // docBufferUpTo == 0 means the block has not been iterated.
+ // docBufferUpTo != 0 means the block has been iterated.
+ docBufferUpto = 1;
+ }
+ }
+ break;
+ }
+
+ return doc;
+ }
+
+ @Override
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
+ if (doc >= upTo) {
+ return;
+ }
+
+ // Handle the current doc separately, it may be on the previous docBuffer.
+ bitSet.set(doc - offset);
+
+ for (; ; ) {
+ if (doc == level0LastDocID) {
+ // refill
+ moveToNextLevel0Block();
+ }
+
+ switch (encoding) {
+ case PACKED:
+ {
+ int start = docBufferUpto;
+ int end = computeBufferEndBoundary(upTo);
+ if (end != 0) {
+ bufferIntoBitSet(start, end, bitSet, offset);
+ doc = docBuffer[end - 1];
+ }
+ docBufferUpto = end;
+ if (end != BLOCK_SIZE) {
+ // Either the block is a tail block, or the block did not fully match, we're done.
+ nextDoc();
+ assert doc >= upTo;
+ return;
+ }
+ }
+ break;
+ case UNARY:
+ {
+ int sourceFrom;
+ if (docBufferUpto == 0) {
+ // start from beginning
+ sourceFrom = 0;
+ } else {
+ // start after the current doc
+ sourceFrom = doc - docBitSetBase + 1;
+ }
+
+ int destFrom = docBitSetBase - offset + sourceFrom;
+
+ assert level0LastDocID != NO_MORE_DOCS;
+ int sourceTo = Math.min(upTo, level0LastDocID + 1) - docBitSetBase;
+
+ if (sourceTo > sourceFrom) {
+ FixedBitSet.orRange(docBitSet, sourceFrom, bitSet, destFrom, sourceTo - sourceFrom);
+ }
+ if (docBitSetBase + sourceTo <= level0LastDocID) {
+ // We stopped before the end of the current bit set, which means that we're done.
+ // Set the current doc before returning.
+ advance(docBitSetBase + sourceTo);
+ return;
+ }
+ doc = level0LastDocID;
+ docBufferUpto = BLOCK_SIZE;
+ }
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {
+ assert needsRefilling == false;
+
+ if (needsFreq == false) {
+ super.nextPostings(upTo, buffer);
+ return;
+ }
+
+ buffer.size = 0;
+ if (doc >= upTo) {
+ return;
+ }
+
+ // Only return docs from the current block
+ // +1 to make BitSetUtil#denseBitsetToArray work, see its java doc.
+ buffer.growNoCopy(BLOCK_SIZE + 1);
+ upTo = (int) Math.min(upTo, level0LastDocID + 1L);
+
+ // Frequencies are decoded lazily, calling freq() makes sure that the freq block is decoded
+ freq();
+
+ int start = docBufferUpto - 1;
+ switch (encoding) {
+ case PACKED:
+ int end = computeBufferEndBoundary(upTo);
+ buffer.size = end - start;
+ System.arraycopy(docBuffer, start, buffer.docs, 0, buffer.size);
+ break;
+ case UNARY:
+ buffer.size =
+ BitSetUtil.denseBitsetToArray(
+ docBitSet, doc - docBitSetBase, upTo - docBitSetBase, docBitSetBase, buffer.docs);
+ break;
+ }
+
+ assert buffer.size > 0;
+ for (int i = 0; i < buffer.size; ++i) {
+ buffer.features[i] = freqBuffer[start + i];
+ }
+
+ advance(upTo);
+ }
+
+ private int computeBufferEndBoundary(int upTo) {
+ if (docBufferSize != 0 && docBuffer[docBufferSize - 1] < upTo) {
+ // All docs in the buffer are under upTo
+ return docBufferSize;
+ } else {
+ // Find the index of the first doc that is greater than or equal to upTo
+ return VectorUtil.findNextGEQ(docBuffer, upTo, docBufferUpto, docBufferSize);
+ }
+ }
+
+ private void bufferIntoBitSet(int start, int end, FixedBitSet bitSet, int offset)
+ throws IOException {
+ // bitSet#set and `doc - offset` get auto-vectorized
+ for (int i = start; i < end; ++i) {
+ int doc = docBuffer[i];
+ bitSet.set(doc - offset);
+ }
+ }
+
+ @Override
+ public int docIDRunEnd() throws IOException {
+ if (encoding == DeltaEncoding.UNARY) {
+ // Note: this assumes that BLOCK_SIZE == 256, this bit of the code would need to be changed
+ // if
+ // the block size was changed.
+ // Hack to avoid compiler warning that both sides of the equal sign are identical.
+ long blockSize = BLOCK_SIZE;
+ assert blockSize == 4 * Long.SIZE;
+ boolean level0IsDense = true;
+ for (int i = 0; i < 4; ++i) {
+ if (docBitSet.getBits()[i] != -1L) {
+ level0IsDense = false;
+ break;
+ }
+ }
+ if (level0IsDense) {
+
+ int level0DocCountUpto = docFreq - docCountLeft;
+ boolean level1IsDense =
+ level1LastDocID - level0LastDocID == level1DocCountUpto - level0DocCountUpto;
+ if (level1IsDense) {
+ return level1LastDocID + 1;
+ }
+
+ return level0LastDocID + 1;
+ }
+ }
+
+ return super.docIDRunEnd();
+ }
+
+ private void skipPositions(int freq) throws IOException {
+ // Skip positions now:
+ int toSkip = posPendingCount - freq;
+ // if (DEBUG) {
+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
+ // }
+
+ final int leftInBlock = BLOCK_SIZE - posBufferUpto;
+ if (toSkip < leftInBlock) {
+ int end = posBufferUpto + toSkip;
+ if (needsPayloads) {
+ payloadByteUpto += sumOverRange(payloadLengthBuffer, posBufferUpto, end);
+ }
+ posBufferUpto = end;
+ } else {
+ toSkip -= leftInBlock;
+ while (toSkip >= BLOCK_SIZE) {
+ assert posIn.getFilePointer() != lastPosBlockFP;
+ PForUtil.skip(posIn);
+
+ if (payIn != null) {
+ if (indexHasPayloads) {
+ // Skip payloadLength block:
+ PForUtil.skip(payIn);
+
+ // Skip payloadBytes block:
+ int numBytes = payIn.readVInt();
+ payIn.seek(payIn.getFilePointer() + numBytes);
+ }
+
+ if (indexHasOffsets) {
+ PForUtil.skip(payIn);
+ PForUtil.skip(payIn);
+ }
+ }
+ toSkip -= BLOCK_SIZE;
+ }
+ refillPositions();
+ if (needsPayloads) {
+ payloadByteUpto = sumOverRange(payloadLengthBuffer, 0, toSkip);
+ }
+ posBufferUpto = toSkip;
+ }
+ }
+
+ private void refillLastPositionBlock() throws IOException {
+ final int count = (int) (totalTermFreq % BLOCK_SIZE);
+ int payloadLength = 0;
+ int offsetLength = 0;
+ payloadByteUpto = 0;
+ for (int i = 0; i < count; i++) {
+ int code = posIn.readVInt();
+ if (indexHasPayloads) {
+ if ((code & 1) != 0) {
+ payloadLength = posIn.readVInt();
+ }
+ if (payloadLengthBuffer != null) { // needs payloads
+ payloadLengthBuffer[i] = payloadLength;
+ posDeltaBuffer[i] = code >>> 1;
+ if (payloadLength != 0) {
+ if (payloadByteUpto + payloadLength > payloadBytes.length) {
+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
+ }
+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
+ payloadByteUpto += payloadLength;
+ }
+ } else {
+ posIn.skipBytes(payloadLength);
+ }
+ } else {
+ posDeltaBuffer[i] = code;
+ }
+
+ if (indexHasOffsets) {
+ int deltaCode = posIn.readVInt();
+ if ((deltaCode & 1) != 0) {
+ offsetLength = posIn.readVInt();
+ }
+ if (offsetStartDeltaBuffer != null) { // needs offsets
+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;
+ offsetLengthBuffer[i] = offsetLength;
+ }
+ }
+ }
+ payloadByteUpto = 0;
+ }
+
+ private void refillOffsetsOrPayloads() throws IOException {
+ if (indexHasPayloads) {
+ if (needsPayloads) {
+ pforUtil.decode(payInUtil, payloadLengthBuffer);
+ int numBytes = payIn.readVInt();
+
+ if (numBytes > payloadBytes.length) {
+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);
+ }
+ payIn.readBytes(payloadBytes, 0, numBytes);
+ } else if (payIn != null) { // needs offsets
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ PForUtil.skip(payIn); // skip over lengths
+ int numBytes = payIn.readVInt(); // read length of payloadBytes
+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
+ }
+ payloadByteUpto = 0;
+ }
+
+ if (indexHasOffsets) {
+ if (needsOffsets) {
+ pforUtil.decode(payInUtil, offsetStartDeltaBuffer);
+ pforUtil.decode(payInUtil, offsetLengthBuffer);
+ } else if (payIn != null) { // needs payloads
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ PForUtil.skip(payIn); // skip over starts
+ PForUtil.skip(payIn); // skip over lengths
+ }
+ }
+ }
+
+ private void refillPositions() throws IOException {
+ if (posIn.getFilePointer() == lastPosBlockFP) {
+ refillLastPositionBlock();
+ return;
+ }
+ pforUtil.decode(posInUtil, posDeltaBuffer);
+
+ if (indexHasOffsetsOrPayloads) {
+ refillOffsetsOrPayloads();
+ }
+ }
+
+ private void accumulatePendingPositions() throws IOException {
+ int freq = freq(); // trigger lazy decoding of freqs
+ posPendingCount += sumOverRange(freqBuffer, posDocBufferUpto, docBufferUpto);
+ posDocBufferUpto = docBufferUpto;
+
+ assert posPendingCount > 0;
+
+ if (posPendingCount > freq) {
+ skipPositions(freq);
+ posPendingCount = freq;
+ }
+ }
+
+ private void accumulatePayloadAndOffsets() {
+ if (needsPayloads) {
+ payloadLength = payloadLengthBuffer[posBufferUpto];
+ payload.bytes = payloadBytes;
+ payload.offset = payloadByteUpto;
+ payload.length = payloadLength;
+ payloadByteUpto += payloadLength;
+ }
+
+ if (needsOffsets) {
+ startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];
+ endOffset = startOffset + offsetLengthBuffer[posBufferUpto];
+ lastStartOffset = startOffset;
+ }
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ if (needsPos == false) {
+ return -1;
+ }
+
+ assert posDocBufferUpto <= docBufferUpto;
+ if (posDocBufferUpto != docBufferUpto) {
+ // First position we're reading on this doc
+ accumulatePendingPositions();
+ position = 0;
+ lastStartOffset = 0;
+ }
+
+ if (posBufferUpto == BLOCK_SIZE) {
+ refillPositions();
+ posBufferUpto = 0;
+ }
+ position += posDeltaBuffer[posBufferUpto];
+
+ if (needsOffsetsOrPayloads) {
+ accumulatePayloadAndOffsets();
+ }
+
+ posBufferUpto++;
+ posPendingCount--;
+ return position;
+ }
+
+ @Override
+ public int startOffset() {
+ if (needsOffsets == false) {
+ return -1;
+ }
+ return startOffset;
+ }
+
+ @Override
+ public int endOffset() {
+ if (needsOffsets == false) {
+ return -1;
+ }
+ return endOffset;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ if (needsPayloads == false || payloadLength == 0) {
+ return null;
+ } else {
+ return payload;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+
+ private final Impacts impacts =
+ new Impacts() {
+
+ private final ByteArrayDataInput scratch = new ByteArrayDataInput();
+
+ @Override
+ public int numLevels() {
+ return indexHasFreq == false || level1LastDocID == NO_MORE_DOCS ? 1 : 2;
+ }
+
+ @Override
+ public int getDocIdUpTo(int level) {
+ if (indexHasFreq == false) {
+ return NO_MORE_DOCS;
+ }
+ if (level == 0) {
+ return level0LastDocID;
+ }
+ return level == 1 ? level1LastDocID : NO_MORE_DOCS;
+ }
+
+ @Override
+ public FreqAndNormBuffer getImpacts(int level) {
+ if (indexHasFreq == false) {
+ // Max freq is 1 since freqs are not indexed
+ impactBuffer.size = 1;
+ impactBuffer.freqs[0] = 1;
+ impactBuffer.norms[0] = 1L;
+ return impactBuffer;
+ }
+ if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
+ return readImpacts(level0SerializedImpacts, impactBuffer);
+ }
+ if (level == 1) {
+ return readImpacts(level1SerializedImpacts, impactBuffer);
+ }
+ impactBuffer.size = 1;
+ impactBuffer.freqs[0] = Integer.MAX_VALUE;
+ impactBuffer.norms[0] = 1L;
+ return impactBuffer;
+ }
+
+ private FreqAndNormBuffer readImpacts(
+ BytesRef serialized, FreqAndNormBuffer impactBuffer) {
+ var scratch = this.scratch;
+ scratch.reset(serialized.bytes, 0, serialized.length);
+ Lucene104PostingsReader.readImpacts(scratch, impactBuffer);
+ return impactBuffer;
+ }
+ };
+
+ @Override
+ public Impacts getImpacts() {
+ assert needsImpacts;
+ return impacts;
+ }
+ }
+
+ /**
+ * @see Lucene104PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)
+ */
+ static int readVInt15(DataInput in) throws IOException {
+ short s = in.readShort();
+ if (s >= 0) {
+ return s;
+ } else {
+ return (s & 0x7FFF) | (in.readVInt() << 15);
+ }
+ }
+
+ /**
+ * @see Lucene104PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)
+ */
+ static long readVLong15(DataInput in) throws IOException {
+ short s = in.readShort();
+ if (s >= 0) {
+ return s;
+ } else {
+ return (s & 0x7FFFL) | (in.readVLong() << 15);
+ }
+ }
+
+ private static void prefetchPostings(IndexInput docIn, IntBlockTermState state)
+ throws IOException {
+ assert state.docFreq > 1; // Singletons are inlined in the terms dict, nothing to prefetch
+ if (docIn.getFilePointer() != state.docStartFP) {
+ // Don't prefetch if the input is already positioned at the right offset, which suggests that
+ // the caller is streaming the entire inverted index (e.g. for merging), let the read-ahead
+ // logic do its work instead. Note that this heuristic doesn't work for terms that have skip
+ // data, since skip data is stored after the last term, but handling all terms that have <256
+ // docs is a good start already.
+ docIn.prefetch(state.docStartFP, 1);
+ }
+ // Note: we don't prefetch positions or offsets, which are less likely to be needed.
+ }
+
+ static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer reuse) {
+ int freq = 0;
+ long norm = 0;
+ int size = 0;
+ while (in.getPosition() < in.length()) {
+ int freqDelta = in.readVInt();
+ if ((freqDelta & 0x01) != 0) {
+ freq += 1 + (freqDelta >>> 1);
+ try {
+ norm += 1 + in.readZLong();
+ } catch (IOException e) {
+ throw new RuntimeException(e); // cannot happen on a BADI
+ }
+ } else {
+ freq += 1 + (freqDelta >>> 1);
+ norm++;
+ }
+ reuse.freqs[size] = freq;
+ reuse.norms[size] = norm;
+ size++;
+ }
+ reuse.size = size;
+ return reuse;
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (docIn != null) {
+ CodecUtil.checksumEntireFile(docIn);
+ }
+ if (posIn != null) {
+ CodecUtil.checksumEntireFile(posIn);
+ }
+ if (payIn != null) {
+ CodecUtil.checksumEntireFile(payIn);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName()
+ + "(positions="
+ + (posIn != null)
+ + ",payloads="
+ + (payIn != null)
+ + ")";
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsWriter.java
new file mode 100644
index 00000000000..ddae0e62755
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104PostingsWriter.java
@@ -0,0 +1,734 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.LEVEL1_MASK;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.META_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.TERMS_CODEC;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
+import org.apache.lucene.codecs.Impact;
+import org.apache.lucene.codecs.PushPostingsWriterBase;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.IntBlockTermState;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+/** Writer for {@link Lucene104PostingsFormat}. */
+public class Lucene104PostingsWriter extends PushPostingsWriterBase {
+
+ static final IntBlockTermState EMPTY_STATE = new IntBlockTermState();
+
+ private final int version;
+
+ IndexOutput metaOut;
+ IndexOutput docOut;
+ IndexOutput posOut;
+ IndexOutput payOut;
+
+ IntBlockTermState lastState;
+
+ // Holds starting file pointers for current term:
+ private long docStartFP;
+ private long posStartFP;
+ private long payStartFP;
+
+ final int[] docDeltaBuffer;
+ final int[] freqBuffer;
+ private int docBufferUpto;
+
+ final int[] posDeltaBuffer;
+ final int[] payloadLengthBuffer;
+ final int[] offsetStartDeltaBuffer;
+ final int[] offsetLengthBuffer;
+ private int posBufferUpto;
+
+ private byte[] payloadBytes;
+ private int payloadByteUpto;
+
+ private int level0LastDocID;
+ private long level0LastPosFP;
+ private long level0LastPayFP;
+
+ private int level1LastDocID;
+ private long level1LastPosFP;
+ private long level1LastPayFP;
+
+ private int docID;
+ private int lastDocID;
+ private int lastPosition;
+ private int lastStartOffset;
+ private int docCount;
+
+ private final PForUtil pforUtil;
+ private final ForUtil forUtil;
+
+ private boolean fieldHasNorms;
+ private NumericDocValues norms;
+ private final CompetitiveImpactAccumulator level0FreqNormAccumulator =
+ new CompetitiveImpactAccumulator();
+ private final CompetitiveImpactAccumulator level1CompetitiveFreqNormAccumulator =
+ new CompetitiveImpactAccumulator();
+
+ private int maxNumImpactsAtLevel0;
+ private int maxImpactNumBytesAtLevel0;
+ private int maxNumImpactsAtLevel1;
+ private int maxImpactNumBytesAtLevel1;
+
+ /** Scratch output that we use to be able to prepend the encoded length, e.g. impacts. */
+ private final ByteBuffersDataOutput scratchOutput = ByteBuffersDataOutput.newResettableInstance();
+
+ /**
+ * Output for a single block. This is useful to be able to prepend skip data before each block,
+ * which can only be computed once the block is encoded. The content is then typically copied to
+ * {@link #level1Output}.
+ */
+ private final ByteBuffersDataOutput level0Output = ByteBuffersDataOutput.newResettableInstance();
+
+ /**
+ * Output for groups of 32 blocks. This is useful to prepend skip data for these 32 blocks, which
+ * can only be done once we have encoded these 32 blocks. The content is then typically copied to
+ * {@link #docCount}.
+ */
+ private final ByteBuffersDataOutput level1Output = ByteBuffersDataOutput.newResettableInstance();
+
+ /**
+ * Reusable FixedBitSet, for dense blocks that are more efficiently stored by storing them as a
+ * bit set than as packed deltas.
+ */
+ // Since we use a bit set when it's more storage-efficient, the bit set cannot have more than
+ // BLOCK_SIZE*32 bits, which is the maximum possible storage requirement with FOR.
+ private final FixedBitSet spareBitSet = new FixedBitSet(BLOCK_SIZE * Integer.SIZE);
+
+ /** Sole public constructor. */
+ public Lucene104PostingsWriter(SegmentWriteState state) throws IOException {
+ this(state, Lucene104PostingsFormat.VERSION_CURRENT);
+ }
+
+ /** Constructor that takes a version. */
+ Lucene104PostingsWriter(SegmentWriteState state, int version) throws IOException {
+ this.version = version;
+ String metaFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.META_EXTENSION);
+ String docFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.DOC_EXTENSION);
+ metaOut = state.directory.createOutput(metaFileName, state.context);
+ IndexOutput posOut = null;
+ IndexOutput payOut = null;
+ try {
+ docOut = state.directory.createOutput(docFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ metaOut, META_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeIndexHeader(
+ docOut, DOC_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
+ forUtil = new ForUtil();
+ pforUtil = new PForUtil(forUtil);
+ if (state.fieldInfos.hasProx()) {
+ posDeltaBuffer = new int[BLOCK_SIZE];
+ String posFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene104PostingsFormat.POS_EXTENSION);
+ posOut = state.directory.createOutput(posFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ posOut, POS_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
+
+ if (state.fieldInfos.hasPayloads()) {
+ payloadBytes = new byte[128];
+ payloadLengthBuffer = new int[BLOCK_SIZE];
+ } else {
+ payloadBytes = null;
+ payloadLengthBuffer = null;
+ }
+
+ if (state.fieldInfos.hasOffsets()) {
+ offsetStartDeltaBuffer = new int[BLOCK_SIZE];
+ offsetLengthBuffer = new int[BLOCK_SIZE];
+ } else {
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ }
+
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ String payFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name,
+ state.segmentSuffix,
+ Lucene104PostingsFormat.PAY_EXTENSION);
+ payOut = state.directory.createOutput(payFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ payOut, PAY_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
+ }
+ } else {
+ posDeltaBuffer = null;
+ payloadLengthBuffer = null;
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ payloadBytes = null;
+ }
+ this.payOut = payOut;
+ this.posOut = posOut;
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, metaOut, docOut, posOut, payOut);
+ throw t;
+ }
+
+ docDeltaBuffer = new int[BLOCK_SIZE];
+ freqBuffer = new int[BLOCK_SIZE];
+ }
+
+ @Override
+ public IntBlockTermState newTermState() {
+ return new IntBlockTermState();
+ }
+
+ @Override
+ public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
+ CodecUtil.writeIndexHeader(
+ termsOut, TERMS_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
+ termsOut.writeVInt(BLOCK_SIZE);
+ }
+
+ @Override
+ public void setField(FieldInfo fieldInfo) {
+ super.setField(fieldInfo);
+ lastState = EMPTY_STATE;
+ fieldHasNorms = fieldInfo.hasNorms();
+ }
+
+ @Override
+ public void startTerm(NumericDocValues norms) {
+ docStartFP = docOut.getFilePointer();
+ if (writePositions) {
+ posStartFP = posOut.getFilePointer();
+ level1LastPosFP = level0LastPosFP = posStartFP;
+ if (writePayloads || writeOffsets) {
+ payStartFP = payOut.getFilePointer();
+ level1LastPayFP = level0LastPayFP = payStartFP;
+ }
+ }
+ lastDocID = -1;
+ level0LastDocID = -1;
+ level1LastDocID = -1;
+ this.norms = norms;
+ if (writeFreqs) {
+ level0FreqNormAccumulator.clear();
+ }
+ }
+
+ @Override
+ public void startDoc(int docID, int termDocFreq) throws IOException {
+ if (docBufferUpto == BLOCK_SIZE) {
+ flushDocBlock(false);
+ docBufferUpto = 0;
+ }
+
+ final int docDelta = docID - lastDocID;
+
+ if (docID < 0 || docDelta <= 0) {
+ throw new CorruptIndexException(
+ "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);
+ }
+
+ docDeltaBuffer[docBufferUpto] = docDelta;
+ if (writeFreqs) {
+ freqBuffer[docBufferUpto] = termDocFreq;
+ }
+
+ this.docID = docID;
+ lastPosition = 0;
+ lastStartOffset = 0;
+
+ if (writeFreqs) {
+ long norm;
+ if (fieldHasNorms) {
+ boolean found = norms.advanceExact(docID);
+ if (found == false) {
+ // This can happen if indexing hits a problem after adding a doc to the
+ // postings but before buffering the norm. Such documents are written
+ // deleted and will go away on the first merge.
+ norm = 1L;
+ } else {
+ norm = norms.longValue();
+ assert norm != 0 : docID;
+ }
+ } else {
+ norm = 1L;
+ }
+
+ level0FreqNormAccumulator.add(termDocFreq, norm);
+ }
+ }
+
+ @Override
+ public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)
+ throws IOException {
+ if (position > IndexWriter.MAX_POSITION) {
+ throw new CorruptIndexException(
+ "position="
+ + position
+ + " is too large (> IndexWriter.MAX_POSITION="
+ + IndexWriter.MAX_POSITION
+ + ")",
+ docOut);
+ }
+ if (position < 0) {
+ throw new CorruptIndexException("position=" + position + " is < 0", docOut);
+ }
+ posDeltaBuffer[posBufferUpto] = position - lastPosition;
+ if (writePayloads) {
+ if (payload == null || payload.length == 0) {
+ // no payload
+ payloadLengthBuffer[posBufferUpto] = 0;
+ } else {
+ payloadLengthBuffer[posBufferUpto] = payload.length;
+ if (payloadByteUpto + payload.length > payloadBytes.length) {
+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);
+ }
+ System.arraycopy(
+ payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);
+ payloadByteUpto += payload.length;
+ }
+ }
+
+ if (writeOffsets) {
+ assert startOffset >= lastStartOffset;
+ assert endOffset >= startOffset;
+ offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
+ offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
+ lastStartOffset = startOffset;
+ }
+
+ posBufferUpto++;
+ lastPosition = position;
+ if (posBufferUpto == BLOCK_SIZE) {
+ pforUtil.encode(posDeltaBuffer, posOut);
+
+ if (writePayloads) {
+ pforUtil.encode(payloadLengthBuffer, payOut);
+ payOut.writeVInt(payloadByteUpto);
+ payOut.writeBytes(payloadBytes, 0, payloadByteUpto);
+ payloadByteUpto = 0;
+ }
+ if (writeOffsets) {
+ pforUtil.encode(offsetStartDeltaBuffer, payOut);
+ pforUtil.encode(offsetLengthBuffer, payOut);
+ }
+ posBufferUpto = 0;
+ }
+ }
+
+ @Override
+ public void finishDoc() {
+ docBufferUpto++;
+ docCount++;
+
+ lastDocID = docID;
+ }
+
+ /**
+ * Special vints that are encoded on 2 bytes if they require 15 bits or less. VInt becomes
+ * especially slow when the number of bytes is variable, so this special layout helps in the case
+ * when the number likely requires 15 bits or less
+ */
+ static void writeVInt15(DataOutput out, int v) throws IOException {
+ assert v >= 0;
+ writeVLong15(out, v);
+ }
+
+ /**
+ * @see #writeVInt15(DataOutput, int)
+ */
+ static void writeVLong15(DataOutput out, long v) throws IOException {
+ assert v >= 0;
+ if ((v & ~0x7FFFL) == 0) {
+ out.writeShort((short) v);
+ } else {
+ out.writeShort((short) (0x8000 | (v & 0x7FFF)));
+ out.writeVLong(v >> 15);
+ }
+ }
+
+ private void flushDocBlock(boolean finishTerm) throws IOException {
+ assert docBufferUpto != 0;
+
+ if (docBufferUpto < BLOCK_SIZE) {
+ assert finishTerm;
+ PostingsUtil.writeVIntBlock(
+ level0Output, docDeltaBuffer, freqBuffer, docBufferUpto, writeFreqs);
+ } else {
+ if (writeFreqs) {
+ List<Impact> impacts = level0FreqNormAccumulator.getCompetitiveFreqNormPairs();
+ if (impacts.size() > maxNumImpactsAtLevel0) {
+ maxNumImpactsAtLevel0 = impacts.size();
+ }
+ writeImpacts(impacts, scratchOutput);
+ assert level0Output.size() == 0;
+ if (scratchOutput.size() > maxImpactNumBytesAtLevel0) {
+ maxImpactNumBytesAtLevel0 = Math.toIntExact(scratchOutput.size());
+ }
+ level0Output.writeVLong(scratchOutput.size());
+ scratchOutput.copyTo(level0Output);
+ scratchOutput.reset();
+ if (writePositions) {
+ level0Output.writeVLong(posOut.getFilePointer() - level0LastPosFP);
+ level0Output.writeByte((byte) posBufferUpto);
+ level0LastPosFP = posOut.getFilePointer();
+
+ if (writeOffsets || writePayloads) {
+ level0Output.writeVLong(payOut.getFilePointer() - level0LastPayFP);
+ level0Output.writeVInt(payloadByteUpto);
+ level0LastPayFP = payOut.getFilePointer();
+ }
+ }
+ }
+ long numSkipBytes = level0Output.size();
+ // Now we need to decide whether to encode block deltas as packed integers (FOR) or unary
+ // codes (bit set). FOR makes #nextDoc() a bit faster while the bit set approach makes
+ // #advance() usually faster and #intoBitSet() much faster. In the end, we make the decision
+ // based on storage requirements, picking the bit set approach whenever it's more
+ // storage-efficient than the next number of bits per value (which effectively slightly biases
+ // towards the bit set approach).
+ int or = 0;
+ for (int i : docDeltaBuffer) {
+ or |= i;
+ }
+ assert or != 0;
+ int bitsPerValue = PackedInts.bitsRequired(or);
+ int docRange = lastDocID - level0LastDocID;
+ assert docRange == Arrays.stream(docDeltaBuffer).sum();
+ int numBitSetLongs = FixedBitSet.bits2words(docRange);
+ int numBitsNextBitsPerValue = Math.min(Integer.SIZE, bitsPerValue + 1) * BLOCK_SIZE;
+ if (docRange == BLOCK_SIZE) {
+ level0Output.writeByte((byte) 0);
+ } else if (numBitsNextBitsPerValue <= docRange) {
+ level0Output.writeByte((byte) bitsPerValue);
+ forUtil.encode(docDeltaBuffer, bitsPerValue, level0Output);
+ } else {
+ // Storing doc deltas is more efficient using unary coding (ie. storing doc IDs as a bit
+ // set)
+ spareBitSet.clear(0, numBitSetLongs << 6);
+ int s = -1;
+ for (int i : docDeltaBuffer) {
+ s += i;
+ spareBitSet.set(s);
+ }
+ // We never use the bit set encoding when it requires more than Integer.SIZE=32 bits per
+ // value. So the bit set cannot have more than BLOCK_SIZE * Integer.SIZE / Long.SIZE = 64
+ // longs, which fits on a byte.
+ assert numBitSetLongs <= BLOCK_SIZE / 2;
+ level0Output.writeByte((byte) -numBitSetLongs);
+ for (int i = 0; i < numBitSetLongs; ++i) {
+ level0Output.writeLong(spareBitSet.getBits()[i]);
+ }
+ }
+
+ if (writeFreqs) {
+ pforUtil.encode(freqBuffer, level0Output);
+ }
+
+ // docID - lastBlockDocID is at least 256, so it can never fit a single byte with a vint
+ // Even if we subtracted 256, only extremely dense blocks would be eligible to a single byte
+ // so let's go with 2 bytes right away
+ writeVInt15(scratchOutput, docID - level0LastDocID);
+ writeVLong15(scratchOutput, level0Output.size());
+ numSkipBytes += scratchOutput.size();
+ level1Output.writeVLong(numSkipBytes);
+ scratchOutput.copyTo(level1Output);
+ scratchOutput.reset();
+ }
+
+ level0Output.copyTo(level1Output);
+ level0Output.reset();
+ level0LastDocID = docID;
+ if (writeFreqs) {
+ level1CompetitiveFreqNormAccumulator.addAll(level0FreqNormAccumulator);
+ level0FreqNormAccumulator.clear();
+ }
+
+ if ((docCount & LEVEL1_MASK) == 0) { // true every 32 blocks (4,096 docs)
+ writeLevel1SkipData();
+ level1LastDocID = docID;
+ level1CompetitiveFreqNormAccumulator.clear();
+ } else if (finishTerm) {
+ level1Output.copyTo(docOut);
+ level1Output.reset();
+ level1CompetitiveFreqNormAccumulator.clear();
+ }
+ }
+
+ private void writeLevel1SkipData() throws IOException {
+ docOut.writeVInt(docID - level1LastDocID);
+ final long level1End;
+ if (writeFreqs) {
+ List<Impact> impacts = level1CompetitiveFreqNormAccumulator.getCompetitiveFreqNormPairs();
+ if (impacts.size() > maxNumImpactsAtLevel1) {
+ maxNumImpactsAtLevel1 = impacts.size();
+ }
+ writeImpacts(impacts, scratchOutput);
+ long numImpactBytes = scratchOutput.size();
+ if (numImpactBytes > maxImpactNumBytesAtLevel1) {
+ maxImpactNumBytesAtLevel1 = Math.toIntExact(numImpactBytes);
+ }
+ if (writePositions) {
+ scratchOutput.writeVLong(posOut.getFilePointer() - level1LastPosFP);
+ scratchOutput.writeByte((byte) posBufferUpto);
+ level1LastPosFP = posOut.getFilePointer();
+ if (writeOffsets || writePayloads) {
+ scratchOutput.writeVLong(payOut.getFilePointer() - level1LastPayFP);
+ scratchOutput.writeVInt(payloadByteUpto);
+ level1LastPayFP = payOut.getFilePointer();
+ }
+ }
+ final long level1Len = 2 * Short.BYTES + scratchOutput.size() + level1Output.size();
+ docOut.writeVLong(level1Len);
+ level1End = docOut.getFilePointer() + level1Len;
+ // There are at most 256 impacts, that require at most 2 bytes each
+ assert numImpactBytes <= Short.MAX_VALUE;
+ // Like impacts plus a few vlongs, still way under the max short value
+ assert scratchOutput.size() + Short.BYTES <= Short.MAX_VALUE;
+ docOut.writeShort((short) (scratchOutput.size() + Short.BYTES));
+ docOut.writeShort((short) numImpactBytes);
+ scratchOutput.copyTo(docOut);
+ scratchOutput.reset();
+ } else {
+ docOut.writeVLong(level1Output.size());
+ level1End = docOut.getFilePointer() + level1Output.size();
+ }
+ level1Output.copyTo(docOut);
+ level1Output.reset();
+ assert docOut.getFilePointer() == level1End : docOut.getFilePointer() + " " + level1End;
+ }
+
+ static void writeImpacts(Collection<Impact> impacts, DataOutput out) throws IOException {
+ Impact previous = new Impact(0, 0);
+ for (Impact impact : impacts) {
+ assert impact.freq > previous.freq;
+ assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
+ int freqDelta = impact.freq - previous.freq - 1;
+ long normDelta = impact.norm - previous.norm - 1;
+ if (normDelta == 0) {
+ // most of time, norm only increases by 1, so we can fold everything in a single byte
+ out.writeVInt(freqDelta << 1);
+ } else {
+ out.writeVInt((freqDelta << 1) | 1);
+ out.writeZLong(normDelta);
+ }
+ previous = impact;
+ }
+ }
+
+ /** Called when we are done adding docs to this term */
+ @Override
+ public void finishTerm(BlockTermState _state) throws IOException {
+ IntBlockTermState state = (IntBlockTermState) _state;
+ assert state.docFreq > 0;
+
+ // TODO: wasteful we are counting this (counting # docs
+ // for this term) in two places?
+ assert state.docFreq == docCount : state.docFreq + " vs " + docCount;
+
+ // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to
+ // it.
+ final int singletonDocID;
+ if (state.docFreq == 1) {
+ // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
+ singletonDocID = docDeltaBuffer[0] - 1;
+ } else {
+ singletonDocID = -1;
+ flushDocBlock(true);
+ }
+
+ final long lastPosBlockOffset;
+
+ if (writePositions) {
+ // totalTermFreq is just total number of positions(or payloads, or offsets)
+ // associated with current term.
+ assert state.totalTermFreq != -1;
+ if (state.totalTermFreq > BLOCK_SIZE) {
+ // record file offset for last pos in last block
+ lastPosBlockOffset = posOut.getFilePointer() - posStartFP;
+ } else {
+ lastPosBlockOffset = -1;
+ }
+ if (posBufferUpto > 0) {
+ assert posBufferUpto < BLOCK_SIZE;
+ // TODO: should we send offsets/payloads to
+ // .pay...? seems wasteful (have to store extra
+ // vLong for low (< BLOCK_SIZE) DF terms = vast vast
+ // majority)
+
+ // vInt encode the remaining positions/payloads/offsets:
+ int lastPayloadLength = -1; // force first payload length to be written
+ int lastOffsetLength = -1; // force first offset length to be written
+ int payloadBytesReadUpto = 0;
+ for (int i = 0; i < posBufferUpto; i++) {
+ final int posDelta = posDeltaBuffer[i];
+ if (writePayloads) {
+ final int payloadLength = payloadLengthBuffer[i];
+ if (payloadLength != lastPayloadLength) {
+ lastPayloadLength = payloadLength;
+ posOut.writeVInt((posDelta << 1) | 1);
+ posOut.writeVInt(payloadLength);
+ } else {
+ posOut.writeVInt(posDelta << 1);
+ }
+
+ if (payloadLength != 0) {
+ posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);
+ payloadBytesReadUpto += payloadLength;
+ }
+ } else {
+ posOut.writeVInt(posDelta);
+ }
+
+ if (writeOffsets) {
+ int delta = offsetStartDeltaBuffer[i];
+ int length = offsetLengthBuffer[i];
+ if (length == lastOffsetLength) {
+ posOut.writeVInt(delta << 1);
+ } else {
+ posOut.writeVInt(delta << 1 | 1);
+ posOut.writeVInt(length);
+ lastOffsetLength = length;
+ }
+ }
+ }
+
+ if (writePayloads) {
+ assert payloadBytesReadUpto == payloadByteUpto;
+ payloadByteUpto = 0;
+ }
+ }
+ } else {
+ lastPosBlockOffset = -1;
+ }
+
+ state.docStartFP = docStartFP;
+ state.posStartFP = posStartFP;
+ state.payStartFP = payStartFP;
+ state.singletonDocID = singletonDocID;
+
+ state.lastPosBlockOffset = lastPosBlockOffset;
+ docBufferUpto = 0;
+ posBufferUpto = 0;
+ lastDocID = -1;
+ docCount = 0;
+ }
+
+ @Override
+ public void encodeTerm(
+ DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)
+ throws IOException {
+ IntBlockTermState state = (IntBlockTermState) _state;
+ if (absolute) {
+ lastState = EMPTY_STATE;
+ assert lastState.docStartFP == 0;
+ }
+
+ if (lastState.singletonDocID != -1
+ && state.singletonDocID != -1
+ && state.docStartFP == lastState.docStartFP) {
+ // With runs of rare values such as ID fields, the increment of pointers in the docs file is
+ // often 0.
+ // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we
+ // encode the delta
+ // between consecutive doc IDs to save space.
+ final long delta = (long) state.singletonDocID - lastState.singletonDocID;
+ out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);
+ } else {
+ out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);
+ if (state.singletonDocID != -1) {
+ out.writeVInt(state.singletonDocID);
+ }
+ }
+
+ if (writePositions) {
+ out.writeVLong(state.posStartFP - lastState.posStartFP);
+ if (writePayloads || writeOffsets) {
+ out.writeVLong(state.payStartFP - lastState.payStartFP);
+ }
+ }
+ if (writePositions) {
+ if (state.lastPosBlockOffset != -1) {
+ out.writeVLong(state.lastPosBlockOffset);
+ }
+ }
+ lastState = state;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // TODO: add a finish() at least to PushBase? DV too...?
+ try {
+ try {
+ if (docOut != null) {
+ CodecUtil.writeFooter(docOut);
+ }
+ if (posOut != null) {
+ CodecUtil.writeFooter(posOut);
+ }
+ if (payOut != null) {
+ CodecUtil.writeFooter(payOut);
+ }
+ if (metaOut != null) {
+ metaOut.writeInt(maxNumImpactsAtLevel0);
+ metaOut.writeInt(maxImpactNumBytesAtLevel0);
+ metaOut.writeInt(maxNumImpactsAtLevel1);
+ metaOut.writeInt(maxImpactNumBytesAtLevel1);
+ metaOut.writeLong(docOut.getFilePointer());
+ if (posOut != null) {
+ metaOut.writeLong(posOut.getFilePointer());
+ if (payOut != null) {
+ metaOut.writeLong(payOut.getFilePointer());
+ }
+ }
+ CodecUtil.writeFooter(metaOut);
+ }
+ } catch (Throwable t) {
+ IOUtils.closeWhileSuppressingExceptions(t, metaOut, docOut, posOut, payOut);
+ throw t;
+ }
+ IOUtils.close(metaOut, docOut, posOut, payOut);
+ } finally {
+ metaOut = docOut = posOut = payOut = null;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PForUtil.java
new file mode 100644
index 00000000000..7985040c599
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PForUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.packed.PackedInts;
+
+/** Utility class to encode sequences of 256 small positive integers. */
+final class PForUtil {
+
+ private static final int MAX_EXCEPTIONS = 7;
+
+ static boolean allEqual(int[] l) {
+ for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {
+ if (l[i] != l[0]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ static {
+ assert ForUtil.BLOCK_SIZE <= 256 : "blocksize must fit in one byte. got " + ForUtil.BLOCK_SIZE;
+ }
+
+ private final ForUtil forUtil;
+
+ PForUtil(ForUtil forUtil) {
+ this.forUtil = forUtil;
+ }
+
+ /** Encode 256 integers from {@code ints} into {@code out}. */
+ void encode(int[] ints, DataOutput out) throws IOException {
+ // histogram of bit widths
+ final int[] histogram = new int[32];
+ int maxBitsRequired = 0;
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ final int v = ints[i];
+ final int bits = PackedInts.bitsRequired(v);
+ histogram[bits]++;
+ maxBitsRequired = Math.max(maxBitsRequired, bits);
+ }
+
+ // We store patch on a byte, so we can't decrease bits by more than 8
+ final int minBits = Math.max(0, maxBitsRequired - 8);
+ int cumulativeExceptions = 0;
+ int patchedBitsRequired = maxBitsRequired;
+ int numExceptions = 0;
+
+ for (int b = maxBitsRequired; b >= minBits; --b) {
+ if (cumulativeExceptions > MAX_EXCEPTIONS) {
+ break;
+ }
+ patchedBitsRequired = b;
+ numExceptions = cumulativeExceptions;
+ cumulativeExceptions += histogram[b];
+ }
+
+ final int maxUnpatchedValue = (1 << patchedBitsRequired) - 1;
+ final byte[] exceptions = new byte[numExceptions * 2];
+ if (numExceptions > 0) {
+ int exceptionCount = 0;
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ if (ints[i] > maxUnpatchedValue) {
+ exceptions[exceptionCount * 2] = (byte) i;
+ exceptions[exceptionCount * 2 + 1] = (byte) (ints[i] >>> patchedBitsRequired);
+ ints[i] &= maxUnpatchedValue;
+ exceptionCount++;
+ }
+ }
+ assert exceptionCount == numExceptions : exceptionCount + " " + numExceptions;
+ }
+
+ if (allEqual(ints) && maxBitsRequired <= 8) {
+ for (int i = 0; i < numExceptions; ++i) {
+ exceptions[2 * i + 1] =
+ (byte) (Byte.toUnsignedInt(exceptions[2 * i + 1]) << patchedBitsRequired);
+ }
+ out.writeByte((byte) (numExceptions << 5));
+ out.writeVInt(ints[0]);
+ } else {
+ final int token = (numExceptions << 5) | patchedBitsRequired;
+ out.writeByte((byte) token);
+ forUtil.encode(ints, patchedBitsRequired, out);
+ }
+ out.writeBytes(exceptions, exceptions.length);
+ }
+
+ /** Decode 256 integers into {@code ints}. */
+ void decode(PostingDecodingUtil pdu, int[] ints) throws IOException {
+ var in = pdu.in;
+ final int token = Byte.toUnsignedInt(in.readByte());
+ final int bitsPerValue = token & 0x1f;
+ if (bitsPerValue == 0) {
+ Arrays.fill(ints, 0, ForUtil.BLOCK_SIZE, in.readVInt());
+ } else {
+ forUtil.decode(bitsPerValue, pdu, ints);
+ }
+ final int numExceptions = token >>> 5;
+ for (int i = 0; i < numExceptions; ++i) {
+ ints[Byte.toUnsignedInt(in.readByte())] |= Byte.toUnsignedInt(in.readByte()) << bitsPerValue;
+ }
+ }
+
+ /** Skip 256 integers. */
+ static void skip(DataInput in) throws IOException {
+ final int token = Byte.toUnsignedInt(in.readByte());
+ final int bitsPerValue = token & 0x1f;
+ final int numExceptions = token >>> 5;
+ if (bitsPerValue == 0) {
+ in.readVLong();
+ in.skipBytes((numExceptions << 1));
+ } else {
+ in.skipBytes(ForUtil.numBytes(bitsPerValue) + (numExceptions << 1));
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingIndexInput.java
similarity index 74%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingIndexInput.java
index e66ed635146..550f0a17659 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingIndexInput.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
import java.io.IOException;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
@@ -32,26 +32,15 @@ public final class PostingIndexInput {
VectorizationProvider.getInstance();
public final ForUtil forUtil;
- public final ForDeltaUtil forDeltaUtil;
private final PostingDecodingUtil postingDecodingUtil;
- public PostingIndexInput(IndexInput in, ForUtil forUtil, ForDeltaUtil forDeltaUtil)
- throws IOException {
+ public PostingIndexInput(IndexInput in, ForUtil forUtil) throws IOException {
this.forUtil = forUtil;
- this.forDeltaUtil = forDeltaUtil;
this.postingDecodingUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
}
- /** Decode 128 integers stored on {@code bitsPerValues} bits per value into {@code ints}. */
+ /** Decode 256 integers stored on {@code bitsPerValues} bits per value into {@code ints}. */
public void decode(int bitsPerValue, int[] ints) throws IOException {
forUtil.decode(bitsPerValue, postingDecodingUtil, ints);
}
-
- /**
- * Decode 128 integers stored on {@code bitsPerValues} bits per value, compute their prefix sum,
- * and store results into {@code ints}.
- */
- public void decodeAndPrefixSum(int bitsPerValue, int base, int[] ints) throws IOException {
- forDeltaUtil.decodeAndPrefixSum(bitsPerValue, postingDecodingUtil, base, ints);
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingsUtil.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingsUtil.java
index e28d59166dc..ed5f25f060c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/PostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py
new file mode 100644
index 00000000000..eb73e4e4e89
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/gen_ForUtil.py
@@ -0,0 +1,328 @@
+#! /usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from math import gcd
+
+"""Code generation for ForUtil.java"""
+
+MAX_SPECIALIZED_BITS_PER_VALUE = 16
+OUTPUT_FILE = "ForUtil.java"
+PRIMITIVE_SIZE = [8, 16, 32]
+HEADER = """// This file has been automatically generated, DO NOT EDIT
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
+import org.apache.lucene.store.DataOutput;
+
+/**
+ * Inspired from https://fulmicoton.com/posts/bitpacking/
+ * Encodes multiple integers in one to get SIMD-like speedups.
+ * If bitsPerValue &lt;= 8 then we pack 4 ints per Java int
+ * else if bitsPerValue &lt;= 16 we pack 2 ints per Java int
+ * else we do scalar operations.
+ */
+public final class ForUtil {
+
+ /** Number of integers per block. */
+ public static final int BLOCK_SIZE = 256;
+ static final int BLOCK_SIZE_LOG2 = 8;
+
+ static int expandMask16(int mask16) {
+ return mask16 | (mask16 << 16);
+ }
+
+ static int expandMask8(int mask8) {
+ return expandMask16(mask8 | (mask8 << 8));
+ }
+
+ static int mask32(int bitsPerValue) {
+ return (1 << bitsPerValue) - 1;
+ }
+
+ static int mask16(int bitsPerValue) {
+ return expandMask16((1 << bitsPerValue) - 1);
+ }
+
+ static int mask8(int bitsPerValue) {
+ return expandMask8((1 << bitsPerValue) - 1);
+ }
+
+ static void expand8(int[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ int l = arr[i];
+ arr[i] = (l >>> 24) & 0xFF;
+ arr[64 + i] = (l >>> 16) & 0xFF;
+ arr[128 + i] = (l >>> 8) & 0xFF;
+ arr[192 + i] = l & 0xFF;
+ }
+ }
+
+ static void collapse8(int[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ arr[i] =
+ (arr[i] << 24)
+ | (arr[64 + i] << 16)
+ | (arr[128 + i] << 8)
+ | arr[192 + i];
+ }
+ }
+
+ static void expand16(int[] arr) {
+ for (int i = 0; i < 128; ++i) {
+ int l = arr[i];
+ arr[i] = (l >>> 16) & 0xFFFF;
+ arr[128 + i] = l & 0xFFFF;
+ }
+ }
+
+ static void collapse16(int[] arr) {
+ for (int i = 0; i < 128; ++i) {
+ arr[i] = (arr[i] << 16) | arr[128 + i];
+ }
+ }
+
+ private final int[] tmp = new int[BLOCK_SIZE];
+
+ /** Encode 256 integers from {@code ints} into {@code out}. */
+ void encode(int[] ints, int bitsPerValue, DataOutput out) throws IOException {
+ final int nextPrimitive;
+ if (bitsPerValue <= 8) {
+ nextPrimitive = 8;
+ collapse8(ints);
+ } else if (bitsPerValue <= 16) {
+ nextPrimitive = 16;
+ collapse16(ints);
+ } else {
+ nextPrimitive = 32;
+ }
+ encode(ints, bitsPerValue, nextPrimitive, out, tmp);
+ }
+
+ static void encode(int[] ints, int bitsPerValue, int primitiveSize, DataOutput out, int[] tmp) throws IOException {
+ final int numInts = BLOCK_SIZE * primitiveSize / Integer.SIZE;
+
+ final int numIntsPerShift = bitsPerValue * 8;
+ int idx = 0;
+ int shift = primitiveSize - bitsPerValue;
+ for (int i = 0; i < numIntsPerShift; ++i) {
+ tmp[i] = ints[idx++] << shift;
+ }
+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
+ for (int i = 0; i < numIntsPerShift; ++i) {
+ tmp[i] |= ints[idx++] << shift;
+ }
+ }
+
+ final int remainingBitsPerInt = shift + bitsPerValue;
+ final int maskRemainingBitsPerInt;
+ if (primitiveSize == 8) {
+ maskRemainingBitsPerInt = MASKS8[remainingBitsPerInt];
+ } else if (primitiveSize == 16) {
+ maskRemainingBitsPerInt = MASKS16[remainingBitsPerInt];
+ } else {
+ maskRemainingBitsPerInt = MASKS32[remainingBitsPerInt];
+ }
+
+ int tmpIdx = 0;
+ int remainingBitsPerValue = bitsPerValue;
+ while (idx < numInts) {
+ if (remainingBitsPerValue >= remainingBitsPerInt) {
+ remainingBitsPerValue -= remainingBitsPerInt;
+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerInt;
+ if (remainingBitsPerValue == 0) {
+ idx++;
+ remainingBitsPerValue = bitsPerValue;
+ }
+ } else {
+ final int mask1, mask2;
+ if (primitiveSize == 8) {
+ mask1 = MASKS8[remainingBitsPerValue];
+ mask2 = MASKS8[remainingBitsPerInt - remainingBitsPerValue];
+ } else if (primitiveSize == 16) {
+ mask1 = MASKS16[remainingBitsPerValue];
+ mask2 = MASKS16[remainingBitsPerInt - remainingBitsPerValue];
+ } else {
+ mask1 = MASKS32[remainingBitsPerValue];
+ mask2 = MASKS32[remainingBitsPerInt - remainingBitsPerValue];
+ }
+ tmp[tmpIdx] |= (ints[idx++] & mask1) << (remainingBitsPerInt - remainingBitsPerValue);
+ remainingBitsPerValue = bitsPerValue - remainingBitsPerInt + remainingBitsPerValue;
+ tmp[tmpIdx++] |= (ints[idx] >>> remainingBitsPerValue) & mask2;
+ }
+ }
+
+ for (int i = 0; i < numIntsPerShift; ++i) {
+ out.writeInt(tmp[i]);
+ }
+ }
+
+ /** Number of bytes required to encode 256 integers of {@code bitsPerValue} bits per value. */
+ static int numBytes(int bitsPerValue) {
+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
+ }
+
+ static void decodeSlow(int bitsPerValue, PostingDecodingUtil pdu, int[] tmp, int[] ints)
+ throws IOException {
+ final int numInts = bitsPerValue << 3;
+ final int mask = MASKS32[bitsPerValue];
+ pdu.splitInts(numInts, ints, 32 - bitsPerValue, 32, mask, tmp, 0, -1);
+ final int remainingBitsPerInt = 32 - bitsPerValue;
+ final int mask32RemainingBitsPerInt = MASKS32[remainingBitsPerInt];
+ int tmpIdx = 0;
+ int remainingBits = remainingBitsPerInt;
+ for (int intsIdx = numInts; intsIdx < BLOCK_SIZE; ++intsIdx) {
+ int b = bitsPerValue - remainingBits;
+ int l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
+ while (b >= remainingBitsPerInt) {
+ b -= remainingBitsPerInt;
+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerInt) << b;
+ }
+ if (b > 0) {
+ l |= (tmp[tmpIdx] >>> (remainingBitsPerInt - b)) & MASKS32[b];
+ remainingBits = remainingBitsPerInt - b;
+ } else {
+ remainingBits = remainingBitsPerInt;
+ }
+ ints[intsIdx] = l;
+ }
+ }
+
+"""
+
+def writeRemainder(bpv, next_primitive, remaining_bits_per_int, o, num_values, f):
+ iteration = 1
+ num_ints = bpv * num_values / remaining_bits_per_int
+ while num_ints % 2 == 0 and num_values % 2 == 0:
+ num_ints /= 2
+ num_values /= 2
+ iteration *= 2
+ f.write(' for (int iter = 0, tmpIdx = 0, intsIdx = %d; iter < %d; ++iter, tmpIdx += %d, intsIdx += %d) {\n' %(o, iteration, num_ints, num_values))
+ i = 0
+ remaining_bits = 0
+ tmp_idx = 0
+ for i in range(int(num_values)):
+ b = bpv
+ if remaining_bits == 0:
+ b -= remaining_bits_per_int
+ f.write(' int l%d = tmp[tmpIdx + %d] << %d;\n' %(i, tmp_idx, b))
+ else:
+ b -= remaining_bits
+ f.write(' int l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits, b))
+ tmp_idx += 1
+ while b >= remaining_bits_per_int:
+ b -= remaining_bits_per_int
+ f.write(' l%d |= tmp[tmpIdx + %d] << %d;\n' %(i, tmp_idx, b))
+ tmp_idx += 1
+ if b > 0:
+ f.write(' l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n' %(i, tmp_idx, remaining_bits_per_int-b, next_primitive, b))
+ remaining_bits = remaining_bits_per_int-b
+ f.write(' ints[intsIdx + %d] = l%d;\n' %(i, i))
+ f.write(' }\n')
+
+
+def writeDecode(bpv, f):
+ next_primitive = 32
+ if bpv <= 8:
+ next_primitive = 8
+ elif bpv <= 16:
+ next_primitive = 16
+ if bpv == next_primitive:
+ f.write(' static void decode%d(PostingDecodingUtil pdu, int[] ints) throws IOException {\n' %bpv)
+ f.write(' pdu.in.readInts(ints, 0, %d);\n' %(bpv*8))
+ else:
+ num_values_per_int = 32 / next_primitive
+ remaining_bits = next_primitive % bpv
+ num_iters = (next_primitive - 1) // bpv
+ o = 8 * bpv * num_iters
+ if remaining_bits == 0:
+ f.write(' static void decode%d(PostingDecodingUtil pdu, int[] ints) throws IOException {\n' %bpv)
+ f.write(' pdu.splitInts(%d, ints, %d, %d, MASK%d_%d, ints, %d, MASK%d_%d);\n' %(bpv*8, next_primitive - bpv, bpv, next_primitive, bpv, o, next_primitive, next_primitive - num_iters * bpv))
+ else:
+ f.write(' static void decode%d(PostingDecodingUtil pdu, int[] tmp, int[] ints) throws IOException {\n' %bpv)
+ f.write(' pdu.splitInts(%d, ints, %d, %d, MASK%d_%d, tmp, 0, MASK%d_%d);\n' %(bpv*8, next_primitive - bpv, bpv, next_primitive, bpv, next_primitive, next_primitive - num_iters * bpv))
+ writeRemainder(bpv, next_primitive, remaining_bits, o, 256/num_values_per_int - o, f)
+ f.write(' }\n')
+
+if __name__ == '__main__':
+ f = open(OUTPUT_FILE, 'w')
+ f.write(HEADER)
+ for primitive_size in PRIMITIVE_SIZE:
+ f.write(' static final int[] MASKS%d = new int[%d];\n' %(primitive_size, primitive_size))
+ f.write('\n')
+ f.write(' static {\n')
+ for primitive_size in PRIMITIVE_SIZE:
+ f.write(' for (int i = 0; i < %d; ++i) {\n' %primitive_size)
+ f.write(' MASKS%d[i] = mask%d(i);\n' %(primitive_size, primitive_size))
+ f.write(' }\n')
+ f.write(' }')
+ f.write("""
+ // mark values in array as final ints to avoid the cost of reading array, arrays should only be
+ // used when the idx is a variable
+""")
+ for primitive_size in PRIMITIVE_SIZE:
+ for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)):
+ f.write(' static final int MASK%d_%d = MASKS%d[%d];\n' %(primitive_size, bpv, primitive_size, bpv))
+
+ f.write("""
+ /** Decode 256 integers into {@code ints}. */
+ void decode(int bitsPerValue, PostingDecodingUtil pdu, int[] ints) throws IOException {
+ switch (bitsPerValue) {
+""")
+ for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):
+ next_primitive = 32
+ if bpv <= 8:
+ next_primitive = 8
+ elif bpv <= 16:
+ next_primitive = 16
+ f.write(' case %d:\n' %bpv)
+ if next_primitive % bpv == 0:
+ f.write(' decode%d(pdu, ints);\n' %bpv)
+ else:
+ f.write(' decode%d(pdu, tmp, ints);\n' %bpv)
+ if next_primitive != 32:
+ f.write(' expand%d(ints);\n' %next_primitive)
+ f.write(' break;\n')
+ f.write(' default:\n')
+ f.write(' decodeSlow(bitsPerValue, pdu, tmp, ints);\n')
+ f.write(' break;\n')
+ f.write(' }\n')
+ f.write(' }\n')
+
+ for i in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):
+ writeDecode(i, f)
+ if i < MAX_SPECIALIZED_BITS_PER_VALUE:
+ f.write('\n')
+
+ f.write('}\n')
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/package-info.java
similarity index 96%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene104/package-info.java
index 0c4bb5a1dde..ebcf8c1e5e4 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/package-info.java
@@ -16,7 +16,7 @@
*/
/**
- * Lucene 10.3 file format.
+ * Lucene 10.4 file format.
*
* <h2>Apache Lucene - Index File Formats</h2>
*
@@ -151,15 +151,15 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term dictionary}. A
+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term dictionary}. A
* dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Frequency data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Frequency data}. For
* each term in the dictionary, the numbers of all the documents that contain that term, and
* the frequency of the term in that document, unless frequencies are omitted ({@link
* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Proximity data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Proximity data}. For
* each term in the dictionary, the positions that the term occurs in each document. Note that
* this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
@@ -414,11 +414,13 @@
* <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector
* format to utilize int8 quantized vectors for float32 vector search.
* <li>In version 9.12, skip data was refactored to have only two levels: every 128 docs and every
- * 4,06 docs, and to be inlined in postings lists. This resulted in a speedup for queries that
- * need skipping, especially conjunctions.
+ * 4,096 docs, and to be inlined in postings lists. This resulted in a speedup for queries
+ * that need skipping, especially conjunctions.
* <li>In version 10.1, block encoding changed to be optimized for int[] storage instead of
* long[].
* <li>In version 10.3, the index of block tree changed to be specialized trie instead of FST.
+ * <li>In version 10.4, the block size was increased from 128 to 256. There are now 256 and 8,192
+ * postings between skip pointers instead of 128 and 4,096.
* </ul>
*
* <a id="Limitations"></a>
@@ -433,4 +435,4 @@
* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
* VInt} values which have no limit. </div>
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
index 48c515b7df5..24864318af5 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
@@ -202,8 +202,8 @@ public abstract class VectorizationProvider {
Set.of(
"org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil",
"org.apache.lucene.util.VectorUtil",
- "org.apache.lucene.codecs.lucene103.Lucene103PostingsReader",
- "org.apache.lucene.codecs.lucene103.PostingIndexInput",
+ "org.apache.lucene.codecs.lucene104.Lucene104PostingsReader",
+ "org.apache.lucene.codecs.lucene104.PostingIndexInput",
"org.apache.lucene.tests.util.TestSysoutsLimits");
private static void ensureCaller() {
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index e478c08e05e..b762fced2ee 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -21,8 +21,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -399,18 +399,18 @@ public class PhraseQuery extends Query {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}.
+ * Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene103PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene104PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
*/
- private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
+ private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 256;
/**
- * Number of simple operations in {@link Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}
+ * Number of simple operations in {@link Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}
* when no seek or buffer refill is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
index 8562ee84c01..457a2cbeefe 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.Comparator;
import java.util.stream.LongStream;
import java.util.stream.StreamSupport;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@@ -32,7 +32,7 @@ import org.apache.lucene.util.VectorUtil;
class ScorerUtil {
private static final Class<?> DEFAULT_IMPACTS_ENUM_CLASS =
- Lucene103PostingsFormat.getImpactsEnumImpl();
+ Lucene104PostingsFormat.getImpactsEnumImpl();
private static final Class<?> DEFAULT_ACCEPT_DOCS_CLASS =
new FixedBitSet(1).asReadOnlyBits().getClass();
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index b85efc3c37f..c13868f1641 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene103.Lucene103Codec
+org.apache.lucene.codecs.lucene104.Lucene104Codec
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 12d44d82f7d..935a82cb22b 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat
+org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestBitSetUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestBitSetUtil.java
similarity index 94%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestBitSetUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestBitSetUtil.java
index dbe4f9f36d8..966c61cfe28 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestBitSetUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestBitSetUtil.java
@@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
-import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat.BLOCK_SIZE;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.FixedBitSet;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestForUtil.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestForUtil.java
index 935eff6e8fd..50386ec1f8e 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
@@ -71,7 +71,7 @@ public class TestForUtil extends LuceneTestCase {
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
PostingDecodingUtil pdu =
- Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ Lucene104PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
ForUtil forUtil = new ForUtil();
for (int i = 0; i < iterations; ++i) {
final int bitsPerValue = in.readByte();
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104PostingsFormat.java
new file mode 100644
index 00000000000..ac1a445ddd5
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestLucene104PostingsFormat.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
+import org.apache.lucene.codecs.Impact;
+import org.apache.lucene.codecs.lucene103.blocktree.FieldReader;
+import org.apache.lucene.codecs.lucene103.blocktree.Stats;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FreqAndNormBuffer;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+public class TestLucene104PostingsFormat extends BasePostingsFormatTestCase {
+
+ @Override
+ protected Codec getCodec() {
+ return TestUtil.alwaysPostingsFormat(new Lucene104PostingsFormat());
+ }
+
+ public void testVInt15() throws IOException {
+ byte[] bytes = new byte[5];
+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ ByteArrayDataInput in = new ByteArrayDataInput();
+ for (int i : new int[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE}) {
+ out.reset(bytes);
+ Lucene104PostingsWriter.writeVInt15(out, i);
+ in.reset(bytes, 0, out.getPosition());
+ assertEquals(i, Lucene104PostingsReader.readVInt15(in));
+ assertEquals(out.getPosition(), in.getPosition());
+ }
+ }
+
+ public void testVLong15() throws IOException {
+ byte[] bytes = new byte[9];
+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ ByteArrayDataInput in = new ByteArrayDataInput();
+ for (long i : new long[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE, Long.MAX_VALUE}) {
+ out.reset(bytes);
+ Lucene104PostingsWriter.writeVLong15(out, i);
+ in.reset(bytes, 0, out.getPosition());
+ assertEquals(i, Lucene104PostingsReader.readVLong15(in));
+ assertEquals(out.getPosition(), in.getPosition());
+ }
+ }
+
+ /** Make sure the final sub-block(s) are not skipped. */
+ public void testFinalBlock() throws Exception {
+ Directory d = newDirectory();
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
+ for (int i = 0; i < 25; i++) {
+ Document doc = new Document();
+ doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));
+ doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ assertEquals(1, r.leaves().size());
+ FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");
+ // We should see exactly two blocks: one root block (prefix empty string) and one block for z*
+ // terms (prefix z):
+ Stats stats = field.getStats();
+ assertEquals(0, stats.floorBlockCount);
+ assertEquals(2, stats.nonFloorBlockCount);
+ r.close();
+ w.close();
+ d.close();
+ }
+
+ public void testImpactSerialization() throws IOException {
+ // omit norms and omit freqs
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
+
+ // omit freqs
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));
+ // omit freqs with very large norms
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));
+
+ // omit norms
+ doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));
+ // omit norms with large freq
+ doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));
+
+ // freqs and norms, basic
+ doTestImpactSerialization(
+ Arrays.asList(
+ new Impact(1, 7L),
+ new Impact(3, 9L),
+ new Impact(7, 10L),
+ new Impact(15, 11L),
+ new Impact(20, 13L),
+ new Impact(28, 14L)));
+
+ // freqs and norms, high values
+ doTestImpactSerialization(
+ Arrays.asList(
+ new Impact(2, 2L),
+ new Impact(10, 10L),
+ new Impact(12, 50L),
+ new Impact(50, -100L),
+ new Impact(1000, -80L),
+ new Impact(1005, -3L)));
+ }
+
+ private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
+ for (Impact impact : impacts) {
+ acc.add(impact.freq, impact.norm);
+ }
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
+ Lucene104PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);
+ }
+ try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
+ byte[] b = new byte[Math.toIntExact(in.length())];
+ in.readBytes(b, 0, b.length);
+ FreqAndNormBuffer impactBuffer = new FreqAndNormBuffer();
+ impactBuffer.growNoCopy(impacts.size() + random().nextInt(3));
+ impactBuffer = Lucene104PostingsReader.readImpacts(new ByteArrayDataInput(b), impactBuffer);
+ List<Impact> impacts2 = new ArrayList<>();
+ for (int i = 0; i < impactBuffer.size; ++i) {
+ impacts2.add(new Impact(impactBuffer.freqs[i], impactBuffer.norms[i]));
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPForUtil.java
similarity index 94%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPForUtil.java
index 61879034ffe..211b7b2427d 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
@@ -41,8 +41,8 @@ public class TestPForUtil extends LuceneTestCase {
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
PostingDecodingUtil pdu =
- Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
- final PForUtil pforUtil = new PForUtil();
+ Lucene104PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ final PForUtil pforUtil = new PForUtil(new ForUtil());
for (int i = 0; i < iterations; ++i) {
if (random().nextInt(5) == 0) {
PForUtil.skip(in);
@@ -90,7 +90,7 @@ public class TestPForUtil extends LuceneTestCase {
private long encodeTestData(int iterations, int[] values, Directory d) throws IOException {
IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
- final PForUtil pforUtil = new PForUtil();
+ final PForUtil pforUtil = new PForUtil(new ForUtil());
for (int i = 0; i < iterations; ++i) {
int[] source = new int[ForUtil.BLOCK_SIZE];
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPostingsUtil.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPostingsUtil.java
index 32edd3608ba..5f4219a33b7 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestPostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene103;
+package org.apache.lucene.codecs.lucene104;
import java.io.IOException;
import org.apache.lucene.store.Directory;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
index b41988631e6..0e9889f54c8 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
@@ -18,7 +18,7 @@ package org.apache.lucene.codecs.lucene90;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DirectoryReader;
@@ -31,7 +31,7 @@ import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION);
+ return new Lucene104Codec(Lucene104Codec.Mode.BEST_COMPRESSION);
}
/**
@@ -42,7 +42,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
for (int i = 0; i < 10; i++) {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setCodec(
- new Lucene103Codec(RandomPicks.randomFrom(random(), Lucene103Codec.Mode.values())));
+ new Lucene104Codec(RandomPicks.randomFrom(random(), Lucene104Codec.Mode.values())));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(new StoredField("field1", "value1"));
@@ -72,7 +72,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
expectThrows(
NullPointerException.class,
() -> {
- new Lucene103Codec(null);
+ new Lucene104Codec(null);
});
expectThrows(
diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java
index 0a9fb4b1ce9..8c96bdaa54d 100644
--- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java
@@ -16,7 +16,7 @@
*/
package org.apache.lucene.internal.vectorization;
-import org.apache.lucene.codecs.lucene103.ForUtil;
+import org.apache.lucene.codecs.lucene104.ForUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java
index ba22fad1ed0..1f6c5d525e4 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/index/TestBpVectorReorderer.java
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinWorkerThread;
-import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -45,6 +43,7 @@ import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TaskExecutor;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.VectorUtil;
/** Tests reordering vector values using Binary Partitioning */
@@ -64,12 +63,7 @@ public class TestBpVectorReorderer extends LuceneTestCase {
private void createQuantizedIndex(Directory dir, List<float[]> vectors) throws IOException {
IndexWriterConfig cfg = new IndexWriterConfig();
cfg.setCodec(
- new Lucene103Codec() {
- @Override
- public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene99HnswScalarQuantizedVectorsFormat(8, 32);
- }
- });
+ TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat(8, 32)));
try (IndexWriter writer = new IndexWriter(dir, cfg)) {
int i = 0;
for (float[] vector : vectors) {
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
index e075aaccc51..68616498a8f 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@@ -261,18 +261,18 @@ class TermIntervalsSource extends IntervalsSource {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}.
+ * Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene103PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene104PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
*/
- private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
+ private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 256;
/**
- * Number of simple operations in {@link Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}
+ * Number of simple operations in {@link Lucene104PostingsReader.BlockPostingsEnum#nextPosition()}
* when no seek or buffer refill is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/suggest/src/java/module-info.java b/lucene/suggest/src/java/module-info.java
index 20542ac5281..c9636811965 100644
--- a/lucene/suggest/src/java/module-info.java
+++ b/lucene/suggest/src/java/module-info.java
@@ -33,7 +33,8 @@ module org.apache.lucene.suggest {
org.apache.lucene.search.suggest.document.Completion90PostingsFormat,
org.apache.lucene.search.suggest.document.Completion99PostingsFormat,
org.apache.lucene.search.suggest.document.Completion912PostingsFormat,
- org.apache.lucene.search.suggest.document.Completion101PostingsFormat;
+ org.apache.lucene.search.suggest.document.Completion101PostingsFormat,
+ org.apache.lucene.search.suggest.document.Completion104PostingsFormat;
provides org.apache.lucene.analysis.TokenFilterFactory with
org.apache.lucene.search.suggest.analyzing.SuggestStopFilterFactory;
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
index 5e419016945..2ac980fe537 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
@@ -19,8 +19,10 @@ package org.apache.lucene.search.suggest.document;
import org.apache.lucene.codecs.PostingsFormat;
/**
- * {@link CompletionPostingsFormat} for {@link
- * org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat}
+ * {@link CompletionPostingsFormat} for {@code
+ * org.apache.lucene.backward_codecs.lucene103.Lucene103PostingsFormat}. This format is only used
+ * for backward-compatibility of the index format and cannot be used to write data, use {@link
+ * Completion104PostingsFormat} on new indices.
*
* @lucene.experimental
*/
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion104PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion104PostingsFormat.java
new file mode 100644
index 00000000000..69b49cf7d7d
--- /dev/null
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion104PostingsFormat.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.suggest.document;
+
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
+
+/**
+ * {@link CompletionPostingsFormat} for {@link Lucene104PostingsFormat}.
+ *
+ * @lucene.experimental
+ */
+public class Completion104PostingsFormat extends CompletionPostingsFormat {
+ /** Creates a {@link Completion104PostingsFormat}. */
+ public Completion104PostingsFormat() {
+ super("Completion104");
+ }
+
+ @Override
+ protected PostingsFormat delegatePostingsFormat() {
+ return PostingsFormat.forName("Lucene104");
+ }
+}
diff --git a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 5c0365616cd..fa4a80aaf67 100644
--- a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -36,3 +36,4 @@ org.apache.lucene.search.suggest.document.Completion90PostingsFormat
org.apache.lucene.search.suggest.document.Completion99PostingsFormat
org.apache.lucene.search.suggest.document.Completion912PostingsFormat
org.apache.lucene.search.suggest.document.Completion101PostingsFormat
+org.apache.lucene.search.suggest.document.Completion104PostingsFormat
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
index 1990baffe72..1548972c814 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
@@ -948,7 +948,7 @@ public class TestSuggestField extends LuceneTestCase {
iwc.setMergePolicy(newLogMergePolicy());
Codec filterCodec =
new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {
- final PostingsFormat postingsFormat = new Completion101PostingsFormat();
+ final PostingsFormat postingsFormat = new Completion104PostingsFormat();
@Override
public PostingsFormat postingsFormat() {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
index 17ceff0d4c2..70e24119da4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
@@ -28,9 +28,9 @@ import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;
import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -38,7 +38,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene103PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
+ * Customized version of {@link Lucene104PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
*/
public final class LuceneFixedGap extends PostingsFormat {
final int termIndexInterval;
@@ -54,7 +54,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene103PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene104PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -91,7 +91,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene103PostingsReader(state);
+ PostingsReaderBase postings = new Lucene104PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
index 741ce732518..719e698eda8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene103PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene104PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval, but forcing high docfreq terms to be indexed
* terms.
*/
@@ -59,7 +59,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene103PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene104PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -100,7 +100,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene103PostingsReader(state);
+ PostingsReaderBase postings = new Lucene104PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
index 93dd141b061..9de1e12c427 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene103PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene104PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval.
*/
public final class LuceneVarGapFixedInterval extends PostingsFormat {
@@ -56,7 +56,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene103PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene104PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -95,7 +95,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene103PostingsReader(state);
+ PostingsReaderBase postings = new Lucene104PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
index e660ccc4478..0c1fa8b4f00 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
@@ -35,10 +35,10 @@ import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;
import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.codecs.memory.FSTTermsReader;
import org.apache.lucene.codecs.memory.FSTTermsWriter;
import org.apache.lucene.index.FieldInfo;
@@ -121,7 +121,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
random.nextInt(); // consume a random for buffersize
- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(state);
final FieldsConsumer fields;
final int t1 = random.nextInt(4);
@@ -289,7 +289,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);
}
- PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(state);
final FieldsProducer fields;
final int t1 = random.nextInt(4);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
index 63f7bcbf2cb..f6abfe2caf6 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsWriter;
import org.apache.lucene.codecs.uniformsplit.BlockDecoder;
import org.apache.lucene.codecs.uniformsplit.BlockEncoder;
import org.apache.lucene.codecs.uniformsplit.IndexDictionary;
@@ -68,7 +68,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState segmentWriteState) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(segmentWriteState);
+ PostingsWriterBase postingsWriter = new Lucene104PostingsWriter(segmentWriteState);
boolean success = false;
try {
FieldsConsumer fieldsConsumer = createFieldsConsumer(segmentWriteState, postingsWriter);
@@ -146,7 +146,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState segmentReadState) throws IOException {
- PostingsReaderBase postingsReader = new Lucene103PostingsReader(segmentReadState);
+ PostingsReaderBase postingsReader = new Lucene104PostingsReader(segmentReadState);
boolean success = false;
try {
FieldsProducer fieldsProducer = createFieldsProducer(segmentReadState, postingsReader);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
index 13db0d34e78..e4c0625e7c5 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
@@ -38,7 +38,7 @@ import java.util.TimeZone;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
@@ -188,9 +188,9 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule {
} else if ("Compressing".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {
codec = CompressingCodec.randomInstance(random);
- } else if ("Lucene100".equals(TEST_CODEC)
- || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene100"))) {
- codec = new Lucene103Codec(RandomPicks.randomFrom(random, Lucene103Codec.Mode.values()));
+ } else if ("Lucene104".equals(TEST_CODEC)
+ || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene104"))) {
+ codec = new Lucene104Codec(RandomPicks.randomFrom(random, Lucene104Codec.Mode.values()));
} else if (!"random".equals(TEST_CODEC)) {
codec = Codec.forName(TEST_CODEC);
} else if ("random".equals(TEST_POSTINGSFORMAT)) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
index 1469ee02bb0..073da0d3a5b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
@@ -62,8 +62,8 @@ import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
-import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104Codec;
+import org.apache.lucene.codecs.lucene104.Lucene104PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
@@ -1327,7 +1327,7 @@ public final class TestUtil {
* different from {@link Codec#getDefault()} because that is randomized.
*/
public static Codec getDefaultCodec() {
- return new Lucene103Codec();
+ return new Lucene104Codec();
}
/**
@@ -1335,7 +1335,7 @@ public final class TestUtil {
* Lucene.
*/
public static PostingsFormat getDefaultPostingsFormat() {
- return new Lucene103PostingsFormat();
+ return new Lucene104PostingsFormat();
}
/**
@@ -1346,7 +1346,7 @@ public final class TestUtil {
*/
public static PostingsFormat getDefaultPostingsFormat(
int minItemsPerBlock, int maxItemsPerBlock) {
- return new Lucene103PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
+ return new Lucene104PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
}
/** Returns a random postings format that supports term ordinals */