Skip to content

Apache Lucene open-source search software

Remove patching for doc blocks. (#12741)

Jakub Slowinski8ae598bae593
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle
index 9ada40c8dde..96f5bfdf974 100644
--- a/gradle/generation/forUtil.gradle
+++ b/gradle/generation/forUtil.gradle
@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
description "Regenerate gen_ForUtil.py"
group "generation"
- def genDir = file("src/java/org/apache/lucene/codecs/lucene90")
+ def genDir = file("src/java/org/apache/lucene/codecs/lucene99")
def genScript = file("${genDir}/gen_ForUtil.py")
def genOutput = file("${genDir}/ForUtil.java")
@@ -44,7 +44,7 @@ configure(project(":lucene:core")) {
configure(project(":lucene:backward-codecs")) {
- task generateForUtilInternal() {
+ task generateForUtil84Internal() {
description "Regenerate gen_ForUtil.py"
group "generation"
@@ -64,6 +64,28 @@ configure(project(":lucene:backward-codecs")) {
}
}
- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
+
+ task generateForUtil90Internal() {
+ description "Regenerate gen_ForUtil.py"
+ group "generation"
+
+ def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene90")
+ def genScript = file("${genDir}/gen_ForUtil.py")
+ def genOutput = file("${genDir}/ForUtil.java")
+
+ inputs.file genScript
+ outputs.file genOutput
+
+ doLast {
+ quietExec {
+ workingDir genDir
+ executable project.externalTool("python3")
+ args = [ '-B', genScript ]
+ }
+ }
+ }
+
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])
}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0ee491bbec5..12b058fe14e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -256,11 +256,12 @@ Optimizations
* GITHUB#12719: Top-level conjunctions that are not sorted by score now have a
specialized bulk scorer. (Adrien Grand)
+* GITHUB#12696: Change Postings back to using FOR in Lucene99PostingsFormat. Freqs, positions and offset keep using PFOR. (Jakub Slowinski)
+
* GITHUB#1052: Faster merging of terms enums. (Adrien Grand)
* GITHUB#11903: Faster sort on high-cardinality string fields. (Adrien Grand)
-
Changes in runtime behavior
---------------------
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil84.json
similarity index 100%
rename from lucene/backward-codecs/src/generated/checksums/generateForUtil.json
rename to lucene/backward-codecs/src/generated/checksums/generateForUtil84.json
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil90.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil90.json
new file mode 100644
index 00000000000..15fd411b1cb
--- /dev/null
+++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil90.json
@@ -0,0 +1,4 @@
+{
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.java": "861cab516c7424e6323831c16f0f521499391a90",
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.py": "b66e2f8012759d6d5ce0d73fabb329ae4a391aa0"
+}
\ No newline at end of file
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 689e0a7d5b4..992ad22a773 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -42,7 +42,8 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
provides org.apache.lucene.codecs.PostingsFormat with
org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,
- org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
+ org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
+ org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.java
index a58c84e314b..c889c8b8626 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java
index b9465985063..77280dcff63 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java
@@ -35,7 +35,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java
similarity index 94%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsFormat.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java
index de0d9c381a8..0fceeb47296 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
@@ -24,7 +24,6 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.MultiLevelSkipListWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.IndexOptions;
@@ -36,7 +35,9 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
/**
- * Lucene 5.0 postings format, which encodes postings in packed integer blocks for fast decode.
+ * Lucene 9.0 postings format, which encodes postings in packed integer blocks for fast decode.
+ *
+ * <p>Note: Lucene90PostingsFormat is now READ ONLY.
*
* <p>Basic idea:
*
@@ -371,30 +372,11 @@ public final class Lucene90PostingsFormat extends PostingsFormat {
// Increment version to change it
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-
- private final int minTermBlockSize;
- private final int maxTermBlockSize;
+ static final int VERSION_CURRENT = 1;
- /** Creates {@code Lucene90PostingsFormat} with default settings. */
+ /** Creates read-only {@code Lucene90PostingsFormat}. */
public Lucene90PostingsFormat() {
- this(
- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
- }
-
- /**
- * Creates {@code Lucene90PostingsFormat} with custom values for {@code minBlockSize} and {@code
- * maxBlockSize} passed to block terms dictionary.
- *
- * @see
- * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
- */
- public Lucene90PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
super("Lucene90");
- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
- this.minTermBlockSize = minTermBlockSize;
- this.maxTermBlockSize = maxTermBlockSize;
}
@Override
@@ -403,20 +385,8 @@ public final class Lucene90PostingsFormat extends PostingsFormat {
}
@Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);
- boolean success = false;
- try {
- FieldsConsumer ret =
- new Lucene90BlockTreeTermsWriter(
- state, postingsWriter, minTermBlockSize, maxTermBlockSize);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) {
+ throw new UnsupportedOperationException("Old codecs may only be used for reading");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.java
index 56f7bafd4fe..190fe5b1d1d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.java
@@ -14,23 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
-import static org.apache.lucene.codecs.lucene90.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.VERSION_START;
+import static org.apache.lucene.backward_codecs.lucene90.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.VERSION_START;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Impacts;
import org.apache.lucene.index.ImpactsEnum;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsWriter.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsWriter.java
similarity index 95%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsWriter.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsWriter.java
index f640d8d2c57..b745d4e1217 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsWriter.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsWriter.java
@@ -14,22 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
-import static org.apache.lucene.codecs.lucene90.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.backward_codecs.lucene90.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90ScoreSkipReader.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90ScoreSkipReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90ScoreSkipReader.java
index 44789a98334..504fd5294e1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90ScoreSkipReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90ScoreSkipReader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import java.util.AbstractList;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipReader.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipReader.java
index da31bd75a80..cd2febadf8f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipReader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipWriter.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipWriter.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipWriter.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipWriter.java
index 0743de226c8..72319eff6a5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipWriter.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipWriter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import java.util.Arrays;
@@ -92,8 +92,7 @@ final class Lucene90SkipWriter extends MultiLevelSkipListWriter {
}
}
- public void setField(
- boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {
+ void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {
this.fieldHasPositions = fieldHasPositions;
this.fieldHasOffsets = fieldHasOffsets;
this.fieldHasPayloads = fieldHasPayloads;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/PForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/PForUtil.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/PForUtil.java
index 211912142a4..69b91660d10 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/PForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/PForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.py
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.py
index a468c590cf9..6c58ce1a9a0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.py
@@ -40,7 +40,7 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java
index 9393d0fa5a1..a354dc0827c 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java
@@ -18,6 +18,7 @@ package org.apache.lucene.backward_codecs.lucene91;
import java.util.Objects;
import org.apache.lucene.backward_codecs.lucene90.Lucene90FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompoundFormat;
@@ -37,7 +38,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java
index 21bc835b61e..6dc0e4edb59 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java
@@ -151,17 +151,17 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.
+ * A dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency
+ * data}. For each term in the dictionary, the numbers of all the documents that contain that
+ * term, and the frequency of the term in that document, unless frequencies are omitted
+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity
+ * data}. For each term in the dictionary, the positions that the term occurs in each
+ * document. Note that this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
* each field in each document, a value is stored that is multiplied into the score for hits
* on that field.
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java
index 28ba9923481..15a61fd0c0d 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java
@@ -18,6 +18,7 @@ package org.apache.lucene.backward_codecs.lucene92;
import java.util.Objects;
import org.apache.lucene.backward_codecs.lucene90.Lucene90FieldInfosFormat;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompoundFormat;
@@ -37,7 +38,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java
index 2e85de218e0..c993b78e9f1 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java
@@ -151,17 +151,17 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.
+ * A dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency
+ * data}. For each term in the dictionary, the numbers of all the documents that contain that
+ * term, and the frequency of the term in that document, unless frequencies are omitted
+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity
+ * data}. For each term in the dictionary, the positions that the term occurs in each
+ * document. Note that this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
* each field in each document, a value is stored that is multiplied into the score for hits
* on that field.
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java
index 324ee042967..d8bc7413feb 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_codecs.lucene94;
import java.util.Objects;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompoundFormat;
@@ -36,7 +37,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java
index ae2bffe0bbb..cd140076a82 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java
@@ -151,17 +151,17 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.
+ * A dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency
+ * data}. For each term in the dictionary, the numbers of all the documents that contain that
+ * term, and the frequency of the term in that document, unless frequencies are omitted
+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity
+ * data}. For each term in the dictionary, the positions that the term occurs in each
+ * document. Note that this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
* each field in each document, a value is stored that is multiplied into the score for hits
* on that field.
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java
index 4e42ff09472..e47de701502 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_codecs.lucene95;
import java.util.Objects;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompoundFormat;
@@ -36,7 +37,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java
index 4ad0f34364a..879704dd842 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java
@@ -151,17 +151,17 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.
+ * A dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency
+ * data}. For each term in the dictionary, the numbers of all the documents that contain that
+ * term, and the frequency of the term in that document, unless frequencies are omitted
+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity
+ * data}. For each term in the dictionary, the positions that the term occurs in each
+ * document. Note that this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
* each field in each document, a value is stored that is multiplied into the score for hits
* on that field.
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 620f7c81a9d..18ac359d2ae 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -15,3 +15,4 @@
org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat
org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat
+org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java
index 7c032f7f58b..240a53f22db 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_codecs.lucene90;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -32,6 +33,7 @@ public class Lucene90RWCodec extends Lucene90Codec {
return defaultKnnVectorsFormat;
}
};
+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();
public Lucene90RWCodec() {
this.defaultKnnVectorsFormat =
@@ -49,4 +51,9 @@ public class Lucene90RWCodec extends Lucene90Codec {
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
+
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return postingsFormat;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
new file mode 100644
index 00000000000..6497bb13d11
--- /dev/null
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene90;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PostingsReaderBase;
+import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.util.IOUtils;
+
+public final class Lucene90RWPostingsFormat extends PostingsFormat {
+
+ /** Size of blocks. */
+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
+
+ /**
+ * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,
+ * but slower skipping in big posting lists.
+ */
+ static final int MAX_SKIP_LEVELS = 10;
+
+ static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";
+ static final String DOC_CODEC = "Lucene90PostingsWriterDoc";
+ static final String POS_CODEC = "Lucene90PostingsWriterPos";
+ static final String PAY_CODEC = "Lucene90PostingsWriterPay";
+
+ private final int minTermBlockSize;
+ private final int maxTermBlockSize;
+
+ /** Creates {@code Lucene90RWPostingsFormat} with default settings. */
+ public Lucene90RWPostingsFormat() {
+ this(
+ Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ }
+
+ public Lucene90RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ super("Lucene90");
+ Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ this.minTermBlockSize = minTermBlockSize;
+ this.maxTermBlockSize = maxTermBlockSize;
+ }
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);
+ boolean success = false;
+ try {
+ FieldsConsumer ret =
+ new Lucene90BlockTreeTermsWriter(
+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+ success = true;
+ return ret;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(postingsWriter);
+ }
+ }
+ }
+
+ @Override
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);
+ boolean success = false;
+ try {
+ FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);
+ success = true;
+ return ret;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(postingsReader);
+ }
+ }
+ }
+
+ /**
+ * Holds all state required for {@link Lucene90PostingsReader} to produce a {@link
+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
+ *
+ * @lucene.internal
+ */
+ public static final class IntBlockTermState extends BlockTermState {
+ /** file pointer to the start of the doc ids enumeration, in .doc file */
+ public long docStartFP;
+
+ /** file pointer to the start of the positions enumeration, in .pos file */
+ public long posStartFP;
+
+ /** file pointer to the start of the payloads enumeration, in .pay file */
+ public long payStartFP;
+
+ /**
+ * file offset for the start of the skip list, relative to docStartFP, if there are more than
+ * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1
+ */
+ public long skipOffset;
+
+ /**
+ * file offset for the last position in the last block, if there are more than {@link
+ * ForUtil#BLOCK_SIZE} positions; otherwise -1
+ *
+ * <p>One might think to use total term frequency to track how many positions are left to read
+ * as we decode the blocks, and decode the last block differently when num_left_positions &lt;
+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip
+ * blocks as the skipper will only tell us new position offset (start of block) and number of
+ * positions to skip for that block, without telling us how many positions it has skipped.
+ */
+ public long lastPosBlockOffset;
+
+ /**
+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
+ * totalTermFreq in this case.
+ */
+ public int singletonDocID;
+
+ /** Sole constructor. */
+ public IntBlockTermState() {
+ skipOffset = -1;
+ lastPosBlockOffset = -1;
+ singletonDocID = -1;
+ }
+
+ @Override
+ public IntBlockTermState clone() {
+ IntBlockTermState other = new IntBlockTermState();
+ other.copyFrom(this);
+ return other;
+ }
+
+ @Override
+ public void copyFrom(TermState _other) {
+ super.copyFrom(_other);
+ IntBlockTermState other = (IntBlockTermState) _other;
+ docStartFP = other.docStartFP;
+ posStartFP = other.posStartFP;
+ payStartFP = other.payStartFP;
+ lastPosBlockOffset = other.lastPosBlockOffset;
+ skipOffset = other.skipOffset;
+ singletonDocID = other.singletonDocID;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString()
+ + " docStartFP="
+ + docStartFP
+ + " posStartFP="
+ + posStartFP
+ + " payStartFP="
+ + payStartFP
+ + " lastPosBlockOffset="
+ + lastPosBlockOffset
+ + " singletonDocID="
+ + singletonDocID;
+ }
+ }
+}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java
index f9bac90b906..ec775b513cd 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java
@@ -16,8 +16,10 @@
*/
package org.apache.lucene.backward_codecs.lucene91;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -32,6 +34,7 @@ public class Lucene91RWCodec extends Lucene91Codec {
}
};
private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();
+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();
public Lucene91RWCodec() {
this.defaultKnnVectorsFormat =
@@ -49,4 +52,9 @@ public class Lucene91RWCodec extends Lucene91Codec {
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
+
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return postingsFormat;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java
index 6008fa5df39..8d086787413 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java
@@ -16,8 +16,10 @@
*/
package org.apache.lucene.backward_codecs.lucene92;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -33,6 +35,7 @@ public class Lucene92RWCodec extends Lucene92Codec {
}
};
private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();
+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();
/** Instantiates a new codec. */
public Lucene92RWCodec() {
@@ -51,4 +54,9 @@ public class Lucene92RWCodec extends Lucene92Codec {
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
+
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return postingsFormat;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java
index 88edd2431f8..678dec4ad19 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java
@@ -16,8 +16,10 @@
*/
package org.apache.lucene.backward_codecs.lucene94;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -32,8 +34,8 @@ public class Lucene94RWCodec extends Lucene94Codec {
return defaultKnnVectorsFormat;
}
};
-
private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();
+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();
/** Instantiates a new codec. */
public Lucene94RWCodec() {
@@ -52,4 +54,9 @@ public class Lucene94RWCodec extends Lucene94Codec {
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
+
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return postingsFormat;
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java
index c8311dfd115..e51a2a391dd 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java
@@ -16,9 +16,11 @@
*/
package org.apache.lucene.backward_codecs.lucene95;
+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;
import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;
import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
@@ -33,6 +35,7 @@ public class Lucene95RWCodec extends Lucene95Codec {
return defaultKnnVectorsFormat;
}
};
+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();
private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();
/** Instantiates a new codec. */
@@ -52,4 +55,9 @@ public class Lucene95RWCodec extends Lucene95Codec {
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfosFormat;
}
+
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return postingsFormat;
+ }
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
index 5ee3451c9e8..d389372d8f8 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
@@ -22,14 +22,14 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
-/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene90PostingsWriter}. */
+/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene99PostingsWriter}. */
public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
private final int minTermBlockSize;
@@ -67,7 +67,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
boolean success = false;
try {
@@ -84,7 +84,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
index c243dd0c0ee..4c355f314f1 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
@@ -24,7 +24,7 @@ import java.util.TreeMap;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
@@ -54,7 +54,7 @@ import org.apache.lucene.util.automaton.TransitionAccessor;
// - or: longer dense skip lists than just next byte?
/**
- * Wraps {@link Lucene90PostingsFormat} format for on-disk storage, but then at read time loads and
+ * Wraps {@link Lucene99PostingsFormat} format for on-disk storage, but then at read time loads and
* stores all terms and postings directly in RAM as byte[], int[].
*
* <p><b>WARNING</b>: This is exceptionally RAM intensive: it makes no effort to compress the
@@ -97,12 +97,12 @@ public final class DirectPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return PostingsFormat.forName("Lucene90").fieldsConsumer(state);
+ return PostingsFormat.forName("Lucene99").fieldsConsumer(state);
}
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- FieldsProducer postings = PostingsFormat.forName("Lucene90").fieldsProducer(state);
+ FieldsProducer postings = PostingsFormat.forName("Lucene99").fieldsProducer(state);
if (state.context.context != IOContext.Context.MERGE) {
FieldsProducer loadedPostings;
try {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
index 4e0fec1587f..0a18b75ac00 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
@@ -22,8 +22,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -41,7 +41,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
boolean success = false;
try {
@@ -57,7 +57,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new FSTTermsReader(state, postingsReader);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
index 6881aedbd50..ae01216ccf6 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
@@ -17,13 +17,13 @@
package org.apache.lucene.codecs.uniformsplit;
-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.BLOCK_SIZE;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.TermState;
@@ -34,7 +34,7 @@ import org.apache.lucene.util.RamUsageEstimator;
/**
* {@link TermState} serializer which encodes each file pointer as a delta relative to a base file
- * pointer. It differs from {@link Lucene90PostingsWriter#encodeTerm} which encodes each file
+ * pointer. It differs from {@link Lucene99PostingsWriter#encodeTerm} which encodes each file
* pointer as a delta relative to the previous file pointer.
*
* <p>It automatically sets the base file pointer to the first valid file pointer for doc start FP,
@@ -95,7 +95,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Writes a {@link BlockTermState} to the provided {@link DataOutput}.
*
- * <p>Simpler variant of {@link Lucene90PostingsWriter#encodeTerm(DataOutput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene99PostingsWriter#encodeTerm(DataOutput, FieldInfo,
* BlockTermState, boolean)}.
*/
public void writeTermState(
@@ -148,7 +148,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Reads a {@link BlockTermState} from the provided {@link DataInput}.
*
- * <p>Simpler variant of {@link Lucene90PostingsReader#decodeTerm(DataInput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene99PostingsReader#decodeTerm(DataInput, FieldInfo,
* BlockTermState, boolean)}.
*
* @param reuse {@link BlockTermState} to reuse; or null to create a new one.
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
index 5ac38e78b8f..8e2b7bb7d54 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -113,7 +113,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
boolean success = false;
try {
FieldsConsumer termsWriter =
@@ -130,7 +130,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
boolean success = false;
try {
FieldsProducer termsReader =
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
index f1df01d09b2..a0fa67508f3 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
@@ -28,7 +28,7 @@
* org.apache.lucene.search.PhraseQuery})
* <li>Quite efficient for {@link org.apache.lucene.search.PrefixQuery}
* <li>Not efficient for spell-check and {@link org.apache.lucene.search.FuzzyQuery}, in this case
- * prefer {@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat}
+ * prefer {@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}
* </ul>
*/
package org.apache.lucene.codecs.uniformsplit;
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
index 69c36e792c0..dde59993f6e 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
@@ -17,7 +17,7 @@
package org.apache.lucene.codecs.lucene90.tests;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
/** Test utility class to create mock {@link IntBlockTermState}. */
public class MockTermStateFactory {
diff --git a/lucene/core/src/generated/checksums/generateForUtil.json b/lucene/core/src/generated/checksums/generateForUtil.json
index 6cd91640bd1..527070d7bf0 100644
--- a/lucene/core/src/generated/checksums/generateForUtil.json
+++ b/lucene/core/src/generated/checksums/generateForUtil.json
@@ -1,4 +1,4 @@
{
- "lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java": "a67f47001bad680d75c57a864a1552a78fc85600",
- "lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py": "3219c6d0289cc64d80dbdbc5db4bd05b8774e797"
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java": "1292ad354d255b1272ffd3db684aa2ddb2bc49ec",
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py": "ab7b63a1b73986cc04e43de1c8f474b97aef5116"
}
\ No newline at end of file
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index 98210a89300..096c965dc98 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -72,7 +72,7 @@ module org.apache.lucene.core {
provides org.apache.lucene.codecs.KnnVectorsFormat with
Lucene99HnswVectorsFormat;
provides org.apache.lucene.codecs.PostingsFormat with
- org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
+ org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
provides org.apache.lucene.index.SortFieldProvider with
org.apache.lucene.search.SortField.Provider,
org.apache.lucene.search.SortedNumericSortField.Provider,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java
new file mode 100644
index 00000000000..2403c997435
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.packed.PackedInts;
+
+/** Utility class to encode/decode increasing sequences of 128 integers. */
+public class ForDeltaUtil {
+
+ // IDENTITY_PLUS_ONE[i] == i+1
+ private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];
+
+ static {
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ IDENTITY_PLUS_ONE[i] = i + 1;
+ }
+ }
+
+ private static void prefixSumOfOnes(long[] arr, long base) {
+ System.arraycopy(IDENTITY_PLUS_ONE, 0, arr, 0, ForUtil.BLOCK_SIZE);
+ // This loop gets auto-vectorized
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ arr[i] += base;
+ }
+ }
+
+ private final ForUtil forUtil;
+
+ ForDeltaUtil(ForUtil forUtil) {
+ this.forUtil = forUtil;
+ }
+
+ /**
+ * Encode deltas of a strictly monotonically increasing sequence of integers. The provided {@code
+ * longs} are expected to be deltas between consecutive values.
+ */
+ void encodeDeltas(long[] longs, DataOutput out) throws IOException {
+ if (longs[0] == 1 && PForUtil.allEqual(longs)) { // happens with very dense postings
+ out.writeByte((byte) 0);
+ } else {
+ long or = 0;
+ for (long l : longs) {
+ or |= l;
+ }
+ assert or != 0;
+ final int bitsPerValue = PackedInts.bitsRequired(or);
+ out.writeByte((byte) bitsPerValue);
+ forUtil.encode(longs, bitsPerValue, out);
+ }
+ }
+
+ /** Decode deltas, compute the prefix sum and add {@code base} to all decoded longs. */
+ void decodeAndPrefixSum(DataInput in, long base, long[] longs) throws IOException {
+ final int bitsPerValue = Byte.toUnsignedInt(in.readByte());
+ if (bitsPerValue == 0) {
+ prefixSumOfOnes(longs, base);
+ } else {
+ forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java
new file mode 100644
index 00000000000..3126cdb02c0
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java
@@ -0,0 +1,1148 @@
+// This file has been automatically generated, DO NOT EDIT
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+// Inspired from https://fulmicoton.com/posts/bitpacking/
+// Encodes multiple integers in a long to get SIMD-like speedups.
+// If bitsPerValue <= 8 then we pack 8 ints per long
+// else if bitsPerValue <= 16 we pack 4 ints per long
+// else we pack 2 ints per long
+final class ForUtil {
+
+ static final int BLOCK_SIZE = 128;
+ private static final int BLOCK_SIZE_LOG2 = 7;
+
+ private static long expandMask32(long mask32) {
+ return mask32 | (mask32 << 32);
+ }
+
+ private static long expandMask16(long mask16) {
+ return expandMask32(mask16 | (mask16 << 16));
+ }
+
+ private static long expandMask8(long mask8) {
+ return expandMask16(mask8 | (mask8 << 8));
+ }
+
+ private static long mask32(int bitsPerValue) {
+ return expandMask32((1L << bitsPerValue) - 1);
+ }
+
+ private static long mask16(int bitsPerValue) {
+ return expandMask16((1L << bitsPerValue) - 1);
+ }
+
+ private static long mask8(int bitsPerValue) {
+ return expandMask8((1L << bitsPerValue) - 1);
+ }
+
+ private static void expand8(long[] arr) {
+ for (int i = 0; i < 16; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 56) & 0xFFL;
+ arr[16 + i] = (l >>> 48) & 0xFFL;
+ arr[32 + i] = (l >>> 40) & 0xFFL;
+ arr[48 + i] = (l >>> 32) & 0xFFL;
+ arr[64 + i] = (l >>> 24) & 0xFFL;
+ arr[80 + i] = (l >>> 16) & 0xFFL;
+ arr[96 + i] = (l >>> 8) & 0xFFL;
+ arr[112 + i] = l & 0xFFL;
+ }
+ }
+
+ private static void expand8To32(long[] arr) {
+ for (int i = 0; i < 16; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 24) & 0x000000FF000000FFL;
+ arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;
+ arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;
+ arr[48 + i] = l & 0x000000FF000000FFL;
+ }
+ }
+
+ private static void collapse8(long[] arr) {
+ for (int i = 0; i < 16; ++i) {
+ arr[i] =
+ (arr[i] << 56)
+ | (arr[16 + i] << 48)
+ | (arr[32 + i] << 40)
+ | (arr[48 + i] << 32)
+ | (arr[64 + i] << 24)
+ | (arr[80 + i] << 16)
+ | (arr[96 + i] << 8)
+ | arr[112 + i];
+ }
+ }
+
+ private static void expand16(long[] arr) {
+ for (int i = 0; i < 32; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 48) & 0xFFFFL;
+ arr[32 + i] = (l >>> 32) & 0xFFFFL;
+ arr[64 + i] = (l >>> 16) & 0xFFFFL;
+ arr[96 + i] = l & 0xFFFFL;
+ }
+ }
+
+ private static void expand16To32(long[] arr) {
+ for (int i = 0; i < 32; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;
+ arr[32 + i] = l & 0x0000FFFF0000FFFFL;
+ }
+ }
+
+ private static void collapse16(long[] arr) {
+ for (int i = 0; i < 32; ++i) {
+ arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];
+ }
+ }
+
+ private static void expand32(long[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ long l = arr[i];
+ arr[i] = l >>> 32;
+ arr[64 + i] = l & 0xFFFFFFFFL;
+ }
+ }
+
+ private static void collapse32(long[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ arr[i] = (arr[i] << 32) | arr[64 + i];
+ }
+ }
+
+ private static void prefixSum8(long[] arr, long base) {
+ expand8To32(arr);
+ prefixSum32(arr, base);
+ }
+
+ private static void prefixSum16(long[] arr, long base) {
+ // We need to move to the next primitive size to avoid overflows
+ expand16To32(arr);
+ prefixSum32(arr, base);
+ }
+
+ private static void prefixSum32(long[] arr, long base) {
+ arr[0] += base << 32;
+ innerPrefixSum32(arr);
+ expand32(arr);
+ final long l = arr[BLOCK_SIZE / 2 - 1];
+ for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) {
+ arr[i] += l;
+ }
+ }
+
+ // For some reason unrolling seems to help
+ private static void innerPrefixSum32(long[] arr) {
+ arr[1] += arr[0];
+ arr[2] += arr[1];
+ arr[3] += arr[2];
+ arr[4] += arr[3];
+ arr[5] += arr[4];
+ arr[6] += arr[5];
+ arr[7] += arr[6];
+ arr[8] += arr[7];
+ arr[9] += arr[8];
+ arr[10] += arr[9];
+ arr[11] += arr[10];
+ arr[12] += arr[11];
+ arr[13] += arr[12];
+ arr[14] += arr[13];
+ arr[15] += arr[14];
+ arr[16] += arr[15];
+ arr[17] += arr[16];
+ arr[18] += arr[17];
+ arr[19] += arr[18];
+ arr[20] += arr[19];
+ arr[21] += arr[20];
+ arr[22] += arr[21];
+ arr[23] += arr[22];
+ arr[24] += arr[23];
+ arr[25] += arr[24];
+ arr[26] += arr[25];
+ arr[27] += arr[26];
+ arr[28] += arr[27];
+ arr[29] += arr[28];
+ arr[30] += arr[29];
+ arr[31] += arr[30];
+ arr[32] += arr[31];
+ arr[33] += arr[32];
+ arr[34] += arr[33];
+ arr[35] += arr[34];
+ arr[36] += arr[35];
+ arr[37] += arr[36];
+ arr[38] += arr[37];
+ arr[39] += arr[38];
+ arr[40] += arr[39];
+ arr[41] += arr[40];
+ arr[42] += arr[41];
+ arr[43] += arr[42];
+ arr[44] += arr[43];
+ arr[45] += arr[44];
+ arr[46] += arr[45];
+ arr[47] += arr[46];
+ arr[48] += arr[47];
+ arr[49] += arr[48];
+ arr[50] += arr[49];
+ arr[51] += arr[50];
+ arr[52] += arr[51];
+ arr[53] += arr[52];
+ arr[54] += arr[53];
+ arr[55] += arr[54];
+ arr[56] += arr[55];
+ arr[57] += arr[56];
+ arr[58] += arr[57];
+ arr[59] += arr[58];
+ arr[60] += arr[59];
+ arr[61] += arr[60];
+ arr[62] += arr[61];
+ arr[63] += arr[62];
+ }
+
+ private final long[] tmp = new long[BLOCK_SIZE / 2];
+
+ /** Encode 128 integers from {@code longs} into {@code out}. */
+ void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
+ final int nextPrimitive;
+ final int numLongs;
+ if (bitsPerValue <= 8) {
+ nextPrimitive = 8;
+ numLongs = BLOCK_SIZE / 8;
+ collapse8(longs);
+ } else if (bitsPerValue <= 16) {
+ nextPrimitive = 16;
+ numLongs = BLOCK_SIZE / 4;
+ collapse16(longs);
+ } else {
+ nextPrimitive = 32;
+ numLongs = BLOCK_SIZE / 2;
+ collapse32(longs);
+ }
+
+ final int numLongsPerShift = bitsPerValue * 2;
+ int idx = 0;
+ int shift = nextPrimitive - bitsPerValue;
+ for (int i = 0; i < numLongsPerShift; ++i) {
+ tmp[i] = longs[idx++] << shift;
+ }
+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
+ for (int i = 0; i < numLongsPerShift; ++i) {
+ tmp[i] |= longs[idx++] << shift;
+ }
+ }
+
+ final int remainingBitsPerLong = shift + bitsPerValue;
+ final long maskRemainingBitsPerLong;
+ if (nextPrimitive == 8) {
+ maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];
+ } else if (nextPrimitive == 16) {
+ maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];
+ } else {
+ maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];
+ }
+
+ int tmpIdx = 0;
+ int remainingBitsPerValue = bitsPerValue;
+ while (idx < numLongs) {
+ if (remainingBitsPerValue >= remainingBitsPerLong) {
+ remainingBitsPerValue -= remainingBitsPerLong;
+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;
+ if (remainingBitsPerValue == 0) {
+ idx++;
+ remainingBitsPerValue = bitsPerValue;
+ }
+ } else {
+ final long mask1, mask2;
+ if (nextPrimitive == 8) {
+ mask1 = MASKS8[remainingBitsPerValue];
+ mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];
+ } else if (nextPrimitive == 16) {
+ mask1 = MASKS16[remainingBitsPerValue];
+ mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];
+ } else {
+ mask1 = MASKS32[remainingBitsPerValue];
+ mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];
+ }
+ tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);
+ remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;
+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;
+ }
+ }
+
+ for (int i = 0; i < numLongsPerShift; ++i) {
+ out.writeLong(tmp[i]);
+ }
+ }
+
+ /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */
+ int numBytes(int bitsPerValue) {
+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
+ }
+
+ private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)
+ throws IOException {
+ final int numLongs = bitsPerValue << 1;
+ in.readLongs(tmp, 0, numLongs);
+ final long mask = MASKS32[bitsPerValue];
+ int longsIdx = 0;
+ int shift = 32 - bitsPerValue;
+ for (; shift >= 0; shift -= bitsPerValue) {
+ shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);
+ longsIdx += numLongs;
+ }
+ final int remainingBitsPerLong = shift + bitsPerValue;
+ final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];
+ int tmpIdx = 0;
+ int remainingBits = remainingBitsPerLong;
+ for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {
+ int b = bitsPerValue - remainingBits;
+ long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
+ while (b >= remainingBitsPerLong) {
+ b -= remainingBitsPerLong;
+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;
+ }
+ if (b > 0) {
+ l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];
+ remainingBits = remainingBitsPerLong - b;
+ } else {
+ remainingBits = remainingBitsPerLong;
+ }
+ longs[longsIdx] = l;
+ }
+ }
+
+ /**
+ * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which
+ * generates SIMD instructions for it in order to shift multiple longs at once.
+ */
+ private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {
+ for (int i = 0; i < count; ++i) {
+ b[bi + i] = (a[i] >>> shift) & mask;
+ }
+ }
+
+ private static final long[] MASKS8 = new long[8];
+ private static final long[] MASKS16 = new long[16];
+ private static final long[] MASKS32 = new long[32];
+
+ static {
+ for (int i = 0; i < 8; ++i) {
+ MASKS8[i] = mask8(i);
+ }
+ for (int i = 0; i < 16; ++i) {
+ MASKS16[i] = mask16(i);
+ }
+ for (int i = 0; i < 32; ++i) {
+ MASKS32[i] = mask32(i);
+ }
+ }
+
+ // mark values in array as final longs to avoid the cost of reading array, arrays should only be
+ // used when the idx is a variable
+ private static final long MASK8_1 = MASKS8[1];
+ private static final long MASK8_2 = MASKS8[2];
+ private static final long MASK8_3 = MASKS8[3];
+ private static final long MASK8_4 = MASKS8[4];
+ private static final long MASK8_5 = MASKS8[5];
+ private static final long MASK8_6 = MASKS8[6];
+ private static final long MASK8_7 = MASKS8[7];
+ private static final long MASK16_1 = MASKS16[1];
+ private static final long MASK16_2 = MASKS16[2];
+ private static final long MASK16_3 = MASKS16[3];
+ private static final long MASK16_4 = MASKS16[4];
+ private static final long MASK16_5 = MASKS16[5];
+ private static final long MASK16_6 = MASKS16[6];
+ private static final long MASK16_7 = MASKS16[7];
+ private static final long MASK16_9 = MASKS16[9];
+ private static final long MASK16_10 = MASKS16[10];
+ private static final long MASK16_11 = MASKS16[11];
+ private static final long MASK16_12 = MASKS16[12];
+ private static final long MASK16_13 = MASKS16[13];
+ private static final long MASK16_14 = MASKS16[14];
+ private static final long MASK16_15 = MASKS16[15];
+ private static final long MASK32_1 = MASKS32[1];
+ private static final long MASK32_2 = MASKS32[2];
+ private static final long MASK32_3 = MASKS32[3];
+ private static final long MASK32_4 = MASKS32[4];
+ private static final long MASK32_5 = MASKS32[5];
+ private static final long MASK32_6 = MASKS32[6];
+ private static final long MASK32_7 = MASKS32[7];
+ private static final long MASK32_8 = MASKS32[8];
+ private static final long MASK32_9 = MASKS32[9];
+ private static final long MASK32_10 = MASKS32[10];
+ private static final long MASK32_11 = MASKS32[11];
+ private static final long MASK32_12 = MASKS32[12];
+ private static final long MASK32_13 = MASKS32[13];
+ private static final long MASK32_14 = MASKS32[14];
+ private static final long MASK32_15 = MASKS32[15];
+ private static final long MASK32_17 = MASKS32[17];
+ private static final long MASK32_18 = MASKS32[18];
+ private static final long MASK32_19 = MASKS32[19];
+ private static final long MASK32_20 = MASKS32[20];
+ private static final long MASK32_21 = MASKS32[21];
+ private static final long MASK32_22 = MASKS32[22];
+ private static final long MASK32_23 = MASKS32[23];
+ private static final long MASK32_24 = MASKS32[24];
+
+ /** Decode 128 integers into {@code longs}. */
+ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
+ switch (bitsPerValue) {
+ case 1:
+ decode1(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 2:
+ decode2(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 3:
+ decode3(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 4:
+ decode4(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 5:
+ decode5(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 6:
+ decode6(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 7:
+ decode7(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 8:
+ decode8(in, tmp, longs);
+ expand8(longs);
+ break;
+ case 9:
+ decode9(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 10:
+ decode10(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 11:
+ decode11(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 12:
+ decode12(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 13:
+ decode13(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 14:
+ decode14(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 15:
+ decode15(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 16:
+ decode16(in, tmp, longs);
+ expand16(longs);
+ break;
+ case 17:
+ decode17(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 18:
+ decode18(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 19:
+ decode19(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 20:
+ decode20(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 21:
+ decode21(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 22:
+ decode22(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 23:
+ decode23(in, tmp, longs);
+ expand32(longs);
+ break;
+ case 24:
+ decode24(in, tmp, longs);
+ expand32(longs);
+ break;
+ default:
+ decodeSlow(bitsPerValue, in, tmp, longs);
+ expand32(longs);
+ break;
+ }
+ }
+
+ /** Delta-decode 128 integers into {@code longs}. */
+ void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs)
+ throws IOException {
+ switch (bitsPerValue) {
+ case 1:
+ decode1(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 2:
+ decode2(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 3:
+ decode3(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 4:
+ decode4(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 5:
+ decode5(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 6:
+ decode6(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 7:
+ decode7(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 8:
+ decode8(in, tmp, longs);
+ prefixSum8(longs, base);
+ break;
+ case 9:
+ decode9(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 10:
+ decode10(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 11:
+ decode11(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 12:
+ decode12(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 13:
+ decode13(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 14:
+ decode14(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 15:
+ decode15(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 16:
+ decode16(in, tmp, longs);
+ prefixSum16(longs, base);
+ break;
+ case 17:
+ decode17(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 18:
+ decode18(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 19:
+ decode19(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 20:
+ decode20(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 21:
+ decode21(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 22:
+ decode22(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 23:
+ decode23(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ case 24:
+ decode24(in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ default:
+ decodeSlow(bitsPerValue, in, tmp, longs);
+ prefixSum32(longs, base);
+ break;
+ }
+ }
+
+ private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 2);
+ shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);
+ shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);
+ shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);
+ shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);
+ shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);
+ shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);
+ shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);
+ shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);
+ }
+
+ private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 4);
+ shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);
+ shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);
+ shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);
+ shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);
+ }
+
+ private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 6);
+ shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);
+ shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {
+ long l0 = (tmp[tmpIdx + 0] & MASK8_2) << 1;
+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;
+ l1 |= (tmp[tmpIdx + 2] & MASK8_2) << 0;
+ longs[longsIdx + 1] = l1;
+ }
+ }
+
+ private static void decode4(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 8);
+ shiftLongs(tmp, 8, longs, 0, 4, MASK8_4);
+ shiftLongs(tmp, 8, longs, 8, 0, MASK8_4);
+ }
+
+ private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 10);
+ shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {
+ long l0 = (tmp[tmpIdx + 0] & MASK8_3) << 2;
+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;
+ l1 |= (tmp[tmpIdx + 2] & MASK8_3) << 1;
+ l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;
+ l2 |= (tmp[tmpIdx + 4] & MASK8_3) << 0;
+ longs[longsIdx + 2] = l2;
+ }
+ }
+
+ private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 12);
+ shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);
+ shiftLongs(tmp, 12, tmp, 0, 0, MASK8_2);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {
+ long l0 = tmp[tmpIdx + 0] << 4;
+ l0 |= tmp[tmpIdx + 1] << 2;
+ l0 |= tmp[tmpIdx + 2] << 0;
+ longs[longsIdx + 0] = l0;
+ }
+ }
+
+ private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 14);
+ shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);
+ shiftLongs(tmp, 14, tmp, 0, 0, MASK8_1);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {
+ long l0 = tmp[tmpIdx + 0] << 6;
+ l0 |= tmp[tmpIdx + 1] << 5;
+ l0 |= tmp[tmpIdx + 2] << 4;
+ l0 |= tmp[tmpIdx + 3] << 3;
+ l0 |= tmp[tmpIdx + 4] << 2;
+ l0 |= tmp[tmpIdx + 5] << 1;
+ l0 |= tmp[tmpIdx + 6] << 0;
+ longs[longsIdx + 0] = l0;
+ }
+ }
+
+ private static void decode8(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(longs, 0, 16);
+ }
+
+ private static void decode9(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 18);
+ shiftLongs(tmp, 18, longs, 0, 7, MASK16_9);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 18; iter < 2; ++iter, tmpIdx += 9, longsIdx += 7) {
+ long l0 = (tmp[tmpIdx + 0] & MASK16_7) << 2;
+ l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;
+ l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;
+ l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;
+ l3 |= (tmp[tmpIdx + 4] & MASK16_7) << 1;
+ l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;
+ l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;
+ longs[longsIdx + 4] = l4;
+ long l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;
+ l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;
+ longs[longsIdx + 5] = l5;
+ long l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;
+ l6 |= (tmp[tmpIdx + 8] & MASK16_7) << 0;
+ longs[longsIdx + 6] = l6;
+ }
+ }
+
+ private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 20);
+ shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {
+ long l0 = (tmp[tmpIdx + 0] & MASK16_6) << 4;
+ l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;
+ l1 |= (tmp[tmpIdx + 2] & MASK16_6) << 2;
+ l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;
+ l2 |= (tmp[tmpIdx + 4] & MASK16_6) << 0;
+ longs[longsIdx + 2] = l2;
+ }
+ }
+
+ private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 22);
+ shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {
+ long l0 = (tmp[tmpIdx + 0] & MASK16_5) << 6;
+ l0 |= (tmp[tmpIdx + 1] & MASK16_5) << 1;
+ l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;
+ l1 |= (tmp[tmpIdx + 3] & MASK16_5) << 2;
+ l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;
+ l2 |= (tmp[tmpIdx + 5] & MASK16_5) << 3;
+ l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;
+ l3 |= (tmp[tmpIdx + 7] & MASK16_5) << 4;
+ l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;
+ l4 |= (tmp[tmpIdx + 9] & MASK16_5) << 5;
+ l4 |= (tmp[tmpIdx + 10] & MASK16_5) << 0;
+ longs[longsIdx + 4] = l4;
+ }
+ }
+
+ private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 24);
+ shiftLongs(tmp, 24, longs, 0, 4, MASK16_12);
+ shiftLongs(tmp, 24, tmp, 0, 0, MASK16_4);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) {
+ long l0 = tmp[tmpIdx + 0] << 8;
+ l0 |= tmp[tmpIdx + 1] << 4;
+ l0 |= tmp[tmpIdx + 2] << 0;
+ longs[longsIdx + 0] = l0;
+ }
+ }
+
+ private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 26);
+ shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {
+ long l0 = (tmp[tmpIdx + 0] & MASK16_3) << 10;
+ l0 |= (tmp[tmpIdx + 1] & MASK16_3) << 7;
+ l0 |= (tmp[tmpIdx + 2] & MASK16_3) << 4;
+ l0 |= (tmp[tmpIdx + 3] & MASK16_3) << 1;
+ l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;
+ l1 |= (tmp[tmpIdx + 5] & MASK16_3) << 8;
+ l1 |= (tmp[tmpIdx + 6] & MASK16_3) << 5;
+ l1 |= (tmp[tmpIdx + 7] & MASK16_3) << 2;
+ l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;
+ l2 |= (tmp[tmpIdx + 9] & MASK16_3) << 9;
+ l2 |= (tmp[tmpIdx + 10] & MASK16_3) << 6;
+ l2 |= (tmp[tmpIdx + 11] & MASK16_3) << 3;
+ l2 |= (tmp[tmpIdx + 12] & MASK16_3) << 0;
+ longs[longsIdx + 2] = l2;
+ }
+ }
+
+ private static void decode14(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 28);
+ shiftLongs(tmp, 28, longs, 0, 2, MASK16_14);
+ shiftLongs(tmp, 28, tmp, 0, 0, MASK16_2);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 28; iter < 4; ++iter, tmpIdx += 7, longsIdx += 1) {
+ long l0 = tmp[tmpIdx + 0] << 12;
+ l0 |= tmp[tmpIdx + 1] << 10;
+ l0 |= tmp[tmpIdx + 2] << 8;
+ l0 |= tmp[tmpIdx + 3] << 6;
+ l0 |= tmp[tmpIdx + 4] << 4;
+ l0 |= tmp[tmpIdx + 5] << 2;
+ l0 |= tmp[tmpIdx + 6] << 0;
+ longs[longsIdx + 0] = l0;
+ }
+ }
+
+ private static void decode15(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 30);
+ shiftLongs(tmp, 30, longs, 0, 1, MASK16_15);
+ shiftLongs(tmp, 30, tmp, 0, 0, MASK16_1);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 30; iter < 2; ++iter, tmpIdx += 15, longsIdx += 1) {
+ long l0 = tmp[tmpIdx + 0] << 14;
+ l0 |= tmp[tmpIdx + 1] << 13;
+ l0 |= tmp[tmpIdx + 2] << 12;
+ l0 |= tmp[tmpIdx + 3] << 11;
+ l0 |= tmp[tmpIdx + 4] << 10;
+ l0 |= tmp[tmpIdx + 5] << 9;
+ l0 |= tmp[tmpIdx + 6] << 8;
+ l0 |= tmp[tmpIdx + 7] << 7;
+ l0 |= tmp[tmpIdx + 8] << 6;
+ l0 |= tmp[tmpIdx + 9] << 5;
+ l0 |= tmp[tmpIdx + 10] << 4;
+ l0 |= tmp[tmpIdx + 11] << 3;
+ l0 |= tmp[tmpIdx + 12] << 2;
+ l0 |= tmp[tmpIdx + 13] << 1;
+ l0 |= tmp[tmpIdx + 14] << 0;
+ longs[longsIdx + 0] = l0;
+ }
+ }
+
+ private static void decode16(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(longs, 0, 32);
+ }
+
+ private static void decode17(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 34);
+ shiftLongs(tmp, 34, longs, 0, 15, MASK32_17);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 34; iter < 2; ++iter, tmpIdx += 17, longsIdx += 15) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_15) << 2;
+ l0 |= (tmp[tmpIdx + 1] >>> 13) & MASK32_2;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK32_13) << 4;
+ l1 |= (tmp[tmpIdx + 2] >>> 11) & MASK32_4;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 2] & MASK32_11) << 6;
+ l2 |= (tmp[tmpIdx + 3] >>> 9) & MASK32_6;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 3] & MASK32_9) << 8;
+ l3 |= (tmp[tmpIdx + 4] >>> 7) & MASK32_8;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 4] & MASK32_7) << 10;
+ l4 |= (tmp[tmpIdx + 5] >>> 5) & MASK32_10;
+ longs[longsIdx + 4] = l4;
+ long l5 = (tmp[tmpIdx + 5] & MASK32_5) << 12;
+ l5 |= (tmp[tmpIdx + 6] >>> 3) & MASK32_12;
+ longs[longsIdx + 5] = l5;
+ long l6 = (tmp[tmpIdx + 6] & MASK32_3) << 14;
+ l6 |= (tmp[tmpIdx + 7] >>> 1) & MASK32_14;
+ longs[longsIdx + 6] = l6;
+ long l7 = (tmp[tmpIdx + 7] & MASK32_1) << 16;
+ l7 |= (tmp[tmpIdx + 8] & MASK32_15) << 1;
+ l7 |= (tmp[tmpIdx + 9] >>> 14) & MASK32_1;
+ longs[longsIdx + 7] = l7;
+ long l8 = (tmp[tmpIdx + 9] & MASK32_14) << 3;
+ l8 |= (tmp[tmpIdx + 10] >>> 12) & MASK32_3;
+ longs[longsIdx + 8] = l8;
+ long l9 = (tmp[tmpIdx + 10] & MASK32_12) << 5;
+ l9 |= (tmp[tmpIdx + 11] >>> 10) & MASK32_5;
+ longs[longsIdx + 9] = l9;
+ long l10 = (tmp[tmpIdx + 11] & MASK32_10) << 7;
+ l10 |= (tmp[tmpIdx + 12] >>> 8) & MASK32_7;
+ longs[longsIdx + 10] = l10;
+ long l11 = (tmp[tmpIdx + 12] & MASK32_8) << 9;
+ l11 |= (tmp[tmpIdx + 13] >>> 6) & MASK32_9;
+ longs[longsIdx + 11] = l11;
+ long l12 = (tmp[tmpIdx + 13] & MASK32_6) << 11;
+ l12 |= (tmp[tmpIdx + 14] >>> 4) & MASK32_11;
+ longs[longsIdx + 12] = l12;
+ long l13 = (tmp[tmpIdx + 14] & MASK32_4) << 13;
+ l13 |= (tmp[tmpIdx + 15] >>> 2) & MASK32_13;
+ longs[longsIdx + 13] = l13;
+ long l14 = (tmp[tmpIdx + 15] & MASK32_2) << 15;
+ l14 |= (tmp[tmpIdx + 16] & MASK32_15) << 0;
+ longs[longsIdx + 14] = l14;
+ }
+ }
+
+ private static void decode18(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 36);
+ shiftLongs(tmp, 36, longs, 0, 14, MASK32_18);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 36; iter < 4; ++iter, tmpIdx += 9, longsIdx += 7) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_14) << 4;
+ l0 |= (tmp[tmpIdx + 1] >>> 10) & MASK32_4;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK32_10) << 8;
+ l1 |= (tmp[tmpIdx + 2] >>> 6) & MASK32_8;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 2] & MASK32_6) << 12;
+ l2 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_12;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 3] & MASK32_2) << 16;
+ l3 |= (tmp[tmpIdx + 4] & MASK32_14) << 2;
+ l3 |= (tmp[tmpIdx + 5] >>> 12) & MASK32_2;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 5] & MASK32_12) << 6;
+ l4 |= (tmp[tmpIdx + 6] >>> 8) & MASK32_6;
+ longs[longsIdx + 4] = l4;
+ long l5 = (tmp[tmpIdx + 6] & MASK32_8) << 10;
+ l5 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_10;
+ longs[longsIdx + 5] = l5;
+ long l6 = (tmp[tmpIdx + 7] & MASK32_4) << 14;
+ l6 |= (tmp[tmpIdx + 8] & MASK32_14) << 0;
+ longs[longsIdx + 6] = l6;
+ }
+ }
+
+ private static void decode19(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 38);
+ shiftLongs(tmp, 38, longs, 0, 13, MASK32_19);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 38; iter < 2; ++iter, tmpIdx += 19, longsIdx += 13) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_13) << 6;
+ l0 |= (tmp[tmpIdx + 1] >>> 7) & MASK32_6;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK32_7) << 12;
+ l1 |= (tmp[tmpIdx + 2] >>> 1) & MASK32_12;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 2] & MASK32_1) << 18;
+ l2 |= (tmp[tmpIdx + 3] & MASK32_13) << 5;
+ l2 |= (tmp[tmpIdx + 4] >>> 8) & MASK32_5;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 4] & MASK32_8) << 11;
+ l3 |= (tmp[tmpIdx + 5] >>> 2) & MASK32_11;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 5] & MASK32_2) << 17;
+ l4 |= (tmp[tmpIdx + 6] & MASK32_13) << 4;
+ l4 |= (tmp[tmpIdx + 7] >>> 9) & MASK32_4;
+ longs[longsIdx + 4] = l4;
+ long l5 = (tmp[tmpIdx + 7] & MASK32_9) << 10;
+ l5 |= (tmp[tmpIdx + 8] >>> 3) & MASK32_10;
+ longs[longsIdx + 5] = l5;
+ long l6 = (tmp[tmpIdx + 8] & MASK32_3) << 16;
+ l6 |= (tmp[tmpIdx + 9] & MASK32_13) << 3;
+ l6 |= (tmp[tmpIdx + 10] >>> 10) & MASK32_3;
+ longs[longsIdx + 6] = l6;
+ long l7 = (tmp[tmpIdx + 10] & MASK32_10) << 9;
+ l7 |= (tmp[tmpIdx + 11] >>> 4) & MASK32_9;
+ longs[longsIdx + 7] = l7;
+ long l8 = (tmp[tmpIdx + 11] & MASK32_4) << 15;
+ l8 |= (tmp[tmpIdx + 12] & MASK32_13) << 2;
+ l8 |= (tmp[tmpIdx + 13] >>> 11) & MASK32_2;
+ longs[longsIdx + 8] = l8;
+ long l9 = (tmp[tmpIdx + 13] & MASK32_11) << 8;
+ l9 |= (tmp[tmpIdx + 14] >>> 5) & MASK32_8;
+ longs[longsIdx + 9] = l9;
+ long l10 = (tmp[tmpIdx + 14] & MASK32_5) << 14;
+ l10 |= (tmp[tmpIdx + 15] & MASK32_13) << 1;
+ l10 |= (tmp[tmpIdx + 16] >>> 12) & MASK32_1;
+ longs[longsIdx + 10] = l10;
+ long l11 = (tmp[tmpIdx + 16] & MASK32_12) << 7;
+ l11 |= (tmp[tmpIdx + 17] >>> 6) & MASK32_7;
+ longs[longsIdx + 11] = l11;
+ long l12 = (tmp[tmpIdx + 17] & MASK32_6) << 13;
+ l12 |= (tmp[tmpIdx + 18] & MASK32_13) << 0;
+ longs[longsIdx + 12] = l12;
+ }
+ }
+
+ private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 40);
+ shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_12) << 8;
+ l0 |= (tmp[tmpIdx + 1] >>> 4) & MASK32_8;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK32_4) << 16;
+ l1 |= (tmp[tmpIdx + 2] & MASK32_12) << 4;
+ l1 |= (tmp[tmpIdx + 3] >>> 8) & MASK32_4;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 3] & MASK32_8) << 12;
+ l2 |= (tmp[tmpIdx + 4] & MASK32_12) << 0;
+ longs[longsIdx + 2] = l2;
+ }
+ }
+
+ private static void decode21(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 42);
+ shiftLongs(tmp, 42, longs, 0, 11, MASK32_21);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 42; iter < 2; ++iter, tmpIdx += 21, longsIdx += 11) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_11) << 10;
+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK32_10;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 1] & MASK32_1) << 20;
+ l1 |= (tmp[tmpIdx + 2] & MASK32_11) << 9;
+ l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_9;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 3] & MASK32_2) << 19;
+ l2 |= (tmp[tmpIdx + 4] & MASK32_11) << 8;
+ l2 |= (tmp[tmpIdx + 5] >>> 3) & MASK32_8;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 5] & MASK32_3) << 18;
+ l3 |= (tmp[tmpIdx + 6] & MASK32_11) << 7;
+ l3 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_7;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 7] & MASK32_4) << 17;
+ l4 |= (tmp[tmpIdx + 8] & MASK32_11) << 6;
+ l4 |= (tmp[tmpIdx + 9] >>> 5) & MASK32_6;
+ longs[longsIdx + 4] = l4;
+ long l5 = (tmp[tmpIdx + 9] & MASK32_5) << 16;
+ l5 |= (tmp[tmpIdx + 10] & MASK32_11) << 5;
+ l5 |= (tmp[tmpIdx + 11] >>> 6) & MASK32_5;
+ longs[longsIdx + 5] = l5;
+ long l6 = (tmp[tmpIdx + 11] & MASK32_6) << 15;
+ l6 |= (tmp[tmpIdx + 12] & MASK32_11) << 4;
+ l6 |= (tmp[tmpIdx + 13] >>> 7) & MASK32_4;
+ longs[longsIdx + 6] = l6;
+ long l7 = (tmp[tmpIdx + 13] & MASK32_7) << 14;
+ l7 |= (tmp[tmpIdx + 14] & MASK32_11) << 3;
+ l7 |= (tmp[tmpIdx + 15] >>> 8) & MASK32_3;
+ longs[longsIdx + 7] = l7;
+ long l8 = (tmp[tmpIdx + 15] & MASK32_8) << 13;
+ l8 |= (tmp[tmpIdx + 16] & MASK32_11) << 2;
+ l8 |= (tmp[tmpIdx + 17] >>> 9) & MASK32_2;
+ longs[longsIdx + 8] = l8;
+ long l9 = (tmp[tmpIdx + 17] & MASK32_9) << 12;
+ l9 |= (tmp[tmpIdx + 18] & MASK32_11) << 1;
+ l9 |= (tmp[tmpIdx + 19] >>> 10) & MASK32_1;
+ longs[longsIdx + 9] = l9;
+ long l10 = (tmp[tmpIdx + 19] & MASK32_10) << 11;
+ l10 |= (tmp[tmpIdx + 20] & MASK32_11) << 0;
+ longs[longsIdx + 10] = l10;
+ }
+ }
+
+ private static void decode22(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 44);
+ shiftLongs(tmp, 44, longs, 0, 10, MASK32_22);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 44; iter < 4; ++iter, tmpIdx += 11, longsIdx += 5) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_10) << 12;
+ l0 |= (tmp[tmpIdx + 1] & MASK32_10) << 2;
+ l0 |= (tmp[tmpIdx + 2] >>> 8) & MASK32_2;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 2] & MASK32_8) << 14;
+ l1 |= (tmp[tmpIdx + 3] & MASK32_10) << 4;
+ l1 |= (tmp[tmpIdx + 4] >>> 6) & MASK32_4;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 4] & MASK32_6) << 16;
+ l2 |= (tmp[tmpIdx + 5] & MASK32_10) << 6;
+ l2 |= (tmp[tmpIdx + 6] >>> 4) & MASK32_6;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 6] & MASK32_4) << 18;
+ l3 |= (tmp[tmpIdx + 7] & MASK32_10) << 8;
+ l3 |= (tmp[tmpIdx + 8] >>> 2) & MASK32_8;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 8] & MASK32_2) << 20;
+ l4 |= (tmp[tmpIdx + 9] & MASK32_10) << 10;
+ l4 |= (tmp[tmpIdx + 10] & MASK32_10) << 0;
+ longs[longsIdx + 4] = l4;
+ }
+ }
+
+ private static void decode23(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 46);
+ shiftLongs(tmp, 46, longs, 0, 9, MASK32_23);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 46; iter < 2; ++iter, tmpIdx += 23, longsIdx += 9) {
+ long l0 = (tmp[tmpIdx + 0] & MASK32_9) << 14;
+ l0 |= (tmp[tmpIdx + 1] & MASK32_9) << 5;
+ l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK32_5;
+ longs[longsIdx + 0] = l0;
+ long l1 = (tmp[tmpIdx + 2] & MASK32_4) << 19;
+ l1 |= (tmp[tmpIdx + 3] & MASK32_9) << 10;
+ l1 |= (tmp[tmpIdx + 4] & MASK32_9) << 1;
+ l1 |= (tmp[tmpIdx + 5] >>> 8) & MASK32_1;
+ longs[longsIdx + 1] = l1;
+ long l2 = (tmp[tmpIdx + 5] & MASK32_8) << 15;
+ l2 |= (tmp[tmpIdx + 6] & MASK32_9) << 6;
+ l2 |= (tmp[tmpIdx + 7] >>> 3) & MASK32_6;
+ longs[longsIdx + 2] = l2;
+ long l3 = (tmp[tmpIdx + 7] & MASK32_3) << 20;
+ l3 |= (tmp[tmpIdx + 8] & MASK32_9) << 11;
+ l3 |= (tmp[tmpIdx + 9] & MASK32_9) << 2;
+ l3 |= (tmp[tmpIdx + 10] >>> 7) & MASK32_2;
+ longs[longsIdx + 3] = l3;
+ long l4 = (tmp[tmpIdx + 10] & MASK32_7) << 16;
+ l4 |= (tmp[tmpIdx + 11] & MASK32_9) << 7;
+ l4 |= (tmp[tmpIdx + 12] >>> 2) & MASK32_7;
+ longs[longsIdx + 4] = l4;
+ long l5 = (tmp[tmpIdx + 12] & MASK32_2) << 21;
+ l5 |= (tmp[tmpIdx + 13] & MASK32_9) << 12;
+ l5 |= (tmp[tmpIdx + 14] & MASK32_9) << 3;
+ l5 |= (tmp[tmpIdx + 15] >>> 6) & MASK32_3;
+ longs[longsIdx + 5] = l5;
+ long l6 = (tmp[tmpIdx + 15] & MASK32_6) << 17;
+ l6 |= (tmp[tmpIdx + 16] & MASK32_9) << 8;
+ l6 |= (tmp[tmpIdx + 17] >>> 1) & MASK32_8;
+ longs[longsIdx + 6] = l6;
+ long l7 = (tmp[tmpIdx + 17] & MASK32_1) << 22;
+ l7 |= (tmp[tmpIdx + 18] & MASK32_9) << 13;
+ l7 |= (tmp[tmpIdx + 19] & MASK32_9) << 4;
+ l7 |= (tmp[tmpIdx + 20] >>> 5) & MASK32_4;
+ longs[longsIdx + 7] = l7;
+ long l8 = (tmp[tmpIdx + 20] & MASK32_5) << 18;
+ l8 |= (tmp[tmpIdx + 21] & MASK32_9) << 9;
+ l8 |= (tmp[tmpIdx + 22] & MASK32_9) << 0;
+ longs[longsIdx + 8] = l8;
+ }
+ }
+
+ private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {
+ in.readLongs(tmp, 0, 48);
+ shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);
+ shiftLongs(tmp, 48, tmp, 0, 0, MASK32_8);
+ for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {
+ long l0 = tmp[tmpIdx + 0] << 16;
+ l0 |= tmp[tmpIdx + 1] << 8;
+ l0 |= tmp[tmpIdx + 2] << 0;
+ longs[longsIdx + 0] = l0;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
index 9633c386ec1..ce6c40f9402 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
@@ -98,7 +98,7 @@ public class Lucene99Codec extends Codec {
super("Lucene99");
this.storedFieldsFormat =
new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);
- this.defaultPostingsFormat = new Lucene90PostingsFormat();
+ this.defaultPostingsFormat = new Lucene99PostingsFormat();
this.defaultDVFormat = new Lucene90DocValuesFormat();
this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
}
@@ -151,7 +151,7 @@ public class Lucene99Codec extends Codec {
/**
* Returns the postings format that should be used for writing new segments of <code>field</code>.
*
- * <p>The default implementation always returns "Lucene90".
+ * <p>The default implementation always returns "Lucene99".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation,
@@ -164,7 +164,7 @@ public class Lucene99Codec extends Codec {
* Returns the docvalues format that should be used for writing new segments of <code>field</code>
* .
*
- * <p>The default implementation always returns "Lucene90".
+ * <p>The default implementation always returns "Lucene99".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java
new file mode 100644
index 00000000000..f233276c6c5
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java
@@ -0,0 +1,518 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.FieldsProducer;
+import org.apache.lucene.codecs.MultiLevelSkipListWriter;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.PostingsReaderBase;
+import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.PackedInts;
+
+/**
+ * Lucene 9.9 postings format, which encodes postings in packed integer blocks for fast decode.
+ *
+ * <p>Basic idea:
+ *
+ * <ul>
+ * <li><b>Packed Blocks and VInt Blocks</b>:
+ * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed
+ * format}): the block size (i.e. number of integers inside block) is fixed (currently 128).
+ * Additionally blocks that are all the same value are encoded in an optimized way.
+ * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block
+ * size is variable.
+ * <li><b>Block structure</b>:
+ * <p>When the postings are long enough, Lucene99PostingsFormat will try to encode most
+ * integer data as a packed block.
+ * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as
+ * two packed blocks, while the remaining 3 are encoded as one VInt block.
+ * <p>Different kinds of data are always encoded separately into different packed blocks, but
+ * may possibly be interleaved into the same VInt block.
+ * <p>This strategy is applied to pairs: &lt;document number, frequency&gt;, &lt;position,
+ * payload length&gt;, &lt;position, offset start, offset length&gt;, and &lt;position,
+ * payload length, offsetstart, offset length&gt;.
+ * <li><b>Skipdata settings</b>:
+ * <p>The structure of skip table is quite similar to previous version of Lucene. Skip
+ * interval is the same as block size, and each skip entry points to the beginning of each
+ * block. However, for the first block, skip data is omitted.
+ * <li><b>Positions, Payloads, and Offsets</b>:
+ * <p>A position is an integer indicating where the term occurs within one document. A payload
+ * is a blob of metadata associated with current position. An offset is a pair of integers
+ * indicating the tokenized start/end offsets for given term in current position: it is
+ * essentially a specialized payload.
+ * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets
+ * (assuming a null payload contributes one count). As mentioned in block structure, it is
+ * possible to encode these three either combined or separately.
+ * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,
+ * position data is separated out as .pos, while payloads and offsets are encoded in .pay
+ * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all
+ * these three are stored interleaved into the .pos (so is payload metadata).
+ * <p>With this strategy, the majority of payload and offset data will be outside .pos file.
+ * So for queries that require only position data, running on a full index with payloads and
+ * offsets, this reduces disk pre-fetches.
+ * </ul>
+ *
+ * <p>Files and detailed format:
+ *
+ * <ul>
+ * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>
+ * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>
+ * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>
+ * <li><code>.pos</code>: <a href="#Positions">Positions</a>
+ * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>
+ * </ul>
+ *
+ * <a id="Termdictionary"></a>
+ *
+ * <dl>
+ * <dd><b>Term Dictionary</b>
+ * <p>The .tim file contains the list of terms in each field along with per-term statistics
+ * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
+ * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on
+ * the format.
+ * <p>NOTE: The term dictionary can plug into different postings implementations: the postings
+ * writer/reader are actually responsible for encoding and decoding the PostingsHeader and
+ * TermMetadata sections described here:
+ * <ul>
+ * <li>PostingsHeader --&gt; Header, PackedBlockSize
+ * <li>TermMetadata --&gt; (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,
+ * PayFPDelta?, SkipFPDelta?
+ * <li>Header, --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>PackedBlockSize, SingletonDocID --&gt; {@link DataOutput#writeVInt VInt}
+ * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --&gt; {@link
+ * DataOutput#writeVLong VLong}
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version
+ * information for the postings.
+ * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width
+ * is determined by the largest integer. Smaller block size result in smaller variance
+ * among width of integers hence smaller indexes. Larger block size result in more
+ * efficient bulk i/o hence better acceleration. This value should always be a multiple
+ * of 64, currently fixed as 128 as a tradeoff. It is also the skip interval used to
+ * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.
+ * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In
+ * particular, it is the difference of file offset between this term's data and previous
+ * term's data (or zero, for the first term in the block).On disk it is stored as the
+ * difference from previous value in sequence.
+ * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.
+ * While PayFPDelta determines the position of this term's &lt;TermPayloads,
+ * TermOffsets?&gt; within the .pay file. Similar to DocFPDelta, it is the difference
+ * between two file positions (or neglected, for fields that omit payloads and offsets).
+ * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last
+ * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or
+ * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to
+ * load following payloads and offsets from .pos instead of .pay. Every time a new block
+ * of positions are to be loaded, the PostingsReader will use this value to check
+ * whether current block is packed format or VInt. When packed format, payloads and
+ * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when
+ * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).
+ * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In
+ * particular, it is the length of the TermFreq data. SkipDelta is only stored if
+ * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene99PostingsFormat).
+ * <li>SingletonDocID is an optimization when a term only appears in one document. In this
+ * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a
+ * VIntBlock at that location, the single document ID is written to the term dictionary.
+ * </ul>
+ * </dl>
+ *
+ * <a id="Termindex"></a>
+ *
+ * <dl>
+ * <dd><b>Term Index</b>
+ * <p>The .tip file contains an index into the term dictionary, so that it can be accessed
+ * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.
+ * </dl>
+ *
+ * <a id="Frequencies"></a>
+ *
+ * <dl>
+ * <dd><b>Frequencies and Skip Data</b>
+ * <p>The .doc file contains the lists of documents which contain each term, along with the
+ * frequency of the term in that document (except when frequencies are omitted: {@link
+ * IndexOptions#DOCS}). It also saves skip data to the beginning of each packed or VInt block,
+ * when the length of document list is larger than packed block size.
+ * <ul>
+ * <li>docFile(.doc) --&gt; Header, &lt;TermFreqs, SkipData?&gt;<sup>TermCount</sup>, Footer
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>TermFreqs --&gt; &lt;PackedBlock&gt; <sup>PackedDocBlockNum</sup>, VIntBlock?
+ * <li>PackedBlock --&gt; PackedDocDeltaBlock, PackedFreqBlock?
+ * <li>VIntBlock --&gt; &lt;DocDelta[,
+ * Freq?]&gt;<sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
+ * <li>SkipData --&gt; &lt;&lt;SkipLevelLength, SkipLevel&gt; <sup>NumSkipLevels-1</sup>,
+ * SkipLevel&gt;, SkipDatum?
+ * <li>SkipLevel --&gt; &lt;SkipDatum&gt; <sup>TrimmedDocFreq/(PackedBlockSize^(Level +
+ * 1))</sup>
+ * <li>SkipDatum --&gt; DocSkip, DocFPSkip, &lt;PosFPSkip, PosBlockOffset, PayLength?,
+ * PayFPSkip?&gt;?, ImpactLength, &lt;CompetitiveFreqDelta, CompetitiveNormDelta?&gt;
+ * <sup>ImpactCount</sup>, SkipChildLevelPointer?
+ * <li>PackedDocDeltaBlock, PackedFreqBlock --&gt; {@link PackedInts PackedInts}
+ * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto,
+ * PayFPSkip, ImpactLength, CompetitiveFreqDelta --&gt; {@link DataOutput#writeVInt
+ * VInt}
+ * <li>CompetitiveNormDelta --&gt; {@link DataOutput#writeZLong ZLong}
+ * <li>SkipChildLevelPointer --&gt; {@link DataOutput#writeVLong VLong}
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>PackedDocDeltaBlock is theoretically generated from two steps:
+ * <ol>
+ * <li>Calculate the difference between each document number and previous one, and get
+ * a d-gaps list (for the first document, use absolute value);
+ * <li>For those d-gaps from first one to
+ * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed
+ * blocks.
+ * </ol>
+ * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.
+ * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a
+ * format that encodes DocDelta and Freq:
+ * <p>DocDelta: if frequencies are indexed, this determines both the document number and
+ * the frequency. In particular, DocDelta/2 is the difference between this document
+ * number and the previous document number (or zero when this is the first document in a
+ * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the
+ * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the
+ * gap (not multiplied by 2) between document numbers and no frequency information is
+ * stored.
+ * <p>For example, the TermFreqs for a term which occurs once in document seven and
+ * three times in document eleven, with frequencies indexed, would be the following
+ * sequence of VInts:
+ * <p>15, 8, 3
+ * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence
+ * of VInts instead:
+ * <p>7,4
+ * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or
+ * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)
+ * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this
+ * trick since the definition of skip entry is a little different from base interface.
+ * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for
+ * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.
+ * However, in Lucene99PostingsFormat, the skip data is saved for
+ * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting
+ * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of
+ * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than
+ * Lucene99SkipWriter.
+ * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or
+ * VInt), it is omitted.
+ * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document
+ * number in the postings (i.e. last document number in each packed block). On disk it
+ * is stored as the difference from previous value in the sequence.
+ * <li>DocFPSkip records the file offsets of each block (excluding )posting at
+ * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.
+ * The file offsets are relative to the start of current term's TermFreqs. On disk it is
+ * also stored as the difference from previous SkipDatum in the sequence.
+ * <li>Since positions and payloads are also block encoded, the skip should skip to related
+ * block first, then fetch the values according to in-block offset. PosFPSkip and
+ * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.
+ * While PosBlockOffset indicates which value to fetch inside the related block
+ * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as
+ * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,
+ * and stored as a difference sequence.
+ * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to
+ * the sum of the payload lengths in the current block up to PosBlockOffset
+ * <li>ImpactLength is the total length of CompetitiveFreqDelta and CompetitiveNormDelta
+ * pairs. CompetitiveFreqDelta and CompetitiveNormDelta are used to safely skip score
+ * calculation for uncompetitive documents; See {@link
+ * org.apache.lucene.codecs.CompetitiveImpactAccumulator} for more details.
+ * </ul>
+ * </dl>
+ *
+ * <a id="Positions"></a>
+ *
+ * <dl>
+ * <dd><b>Positions</b>
+ * <p>The .pos file contains the lists of positions that each term occurs at within documents.
+ * It also sometimes stores part of payloads and offsets for speedup.
+ * <ul>
+ * <li>PosFile(.pos) --&gt; Header, &lt;TermPositions&gt; <sup>TermCount</sup>, Footer
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>TermPositions --&gt; &lt;PackedPosDeltaBlock&gt; <sup>PackedPosBlockNum</sup>,
+ * VIntBlock?
+ * <li>VIntBlock --&gt; &lt;PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,
+ * OffsetLength?&gt;<sup>PosVIntCount</sup>
+ * <li>PackedPosDeltaBlock --&gt; {@link PackedInts PackedInts}
+ * <li>PositionDelta, OffsetDelta, OffsetLength --&gt; {@link DataOutput#writeVInt VInt}
+ * <li>PayloadData --&gt; {@link DataOutput#writeByte byte}<sup>PayLength</sup>
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and
+ * position values for each term document pair are incremental, and ordered by document
+ * number.
+ * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,
+ * payloads or offsets. In particular, PackedPosBlockNum =
+ * floor(totalTermFreq/PackedBlockSize)
+ * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,
+ * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize
+ * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock
+ * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.
+ * <li>PositionDelta is, if payloads are disabled for the term's field, the difference
+ * between the position of the current occurrence in the document and the previous
+ * occurrence (or zero, if this is the first occurrence in this document). If payloads
+ * are enabled for the term's field, then PositionDelta/2 is the difference between the
+ * current and the previous position. If payloads are enabled and PositionDelta is odd,
+ * then PayloadLength is stored, indicating the length of the payload at the current
+ * term position.
+ * <li>For example, the TermPositions for a term which occurs as the fourth term in one
+ * document, and as the fifth and ninth term in a subsequent document, would be the
+ * following sequence of VInts (payloads disabled):
+ * <p>4, 5, 4
+ * <li>PayloadData is metadata associated with the current term position. If PayloadLength
+ * is stored at the current position, then it indicates the length of this payload. If
+ * PayloadLength is not stored, then this payload has the same length as the payload at
+ * the previous position.
+ * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous
+ * occurrence (or zero, if this is the first occurrence in this document). If
+ * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous
+ * occurrence and an OffsetLength follows. Offset data is only written for {@link
+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.
+ * </ul>
+ * </dl>
+ *
+ * <a id="Payloads"></a>
+ *
+ * <dl>
+ * <dd><b>Payloads and Offsets</b>
+ * <p>The .pay file will store payloads and offsets associated with certain term-document
+ * positions. Some payloads and offsets will be separated out into .pos file, for performance
+ * reasons.
+ * <ul>
+ * <li>PayFile(.pay): --&gt; Header, &lt;TermPayloads?, TermOffsets?&gt;
+ * <sup>TermCount</sup>, Footer
+ * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
+ * <li>TermPayloads --&gt; &lt;PackedPayLengthBlock, SumPayLength, PayData&gt;
+ * <sup>PackedPayBlockNum</sup>
+ * <li>TermOffsets --&gt; &lt;PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock&gt;
+ * <sup>PackedPayBlockNum</sup>
+ * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock --&gt;
+ * {@link PackedInts PackedInts}
+ * <li>SumPayLength --&gt; {@link DataOutput#writeVInt VInt}
+ * <li>PayData --&gt; {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>
+ * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
+ * </ul>
+ * <p>Notes:
+ * <ul>
+ * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that
+ * part of payload/offsets are stored in .pos.
+ * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is
+ * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip
+ * Data</a>. While PackedStartDeltaBlock follows a same procedure as
+ * PackedDocDeltaBlock.
+ * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also
+ * synonym for PackedOffsetBlockNum.
+ * <li>SumPayLength is the total length of payloads written within one block, should be the
+ * sum of PayLengths in one packed block.
+ * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the
+ * current position.
+ * </ul>
+ * </dl>
+ *
+ * @lucene.experimental
+ */
+public final class Lucene99PostingsFormat extends PostingsFormat {
+
+ /**
+ * Filename extension for document number, frequencies, and skip data. See chapter: <a
+ * href="#Frequencies">Frequencies and Skip Data</a>
+ */
+ public static final String DOC_EXTENSION = "doc";
+
+ /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */
+ public static final String POS_EXTENSION = "pos";
+
+ /**
+ * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and
+ * Offsets</a>
+ */
+ public static final String PAY_EXTENSION = "pay";
+
+ /** Size of blocks. */
+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
+
+ /**
+ * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,
+ * but slower skipping in big posting lists.
+ */
+ static final int MAX_SKIP_LEVELS = 10;
+
+ static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";
+ static final String DOC_CODEC = "Lucene99PostingsWriterDoc";
+ static final String POS_CODEC = "Lucene99PostingsWriterPos";
+ static final String PAY_CODEC = "Lucene99PostingsWriterPay";
+
+ // Increment version to change it
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = VERSION_START;
+
+ private final int minTermBlockSize;
+ private final int maxTermBlockSize;
+
+ /** Creates {@code Lucene99PostingsFormat} with default settings. */
+ public Lucene99PostingsFormat() {
+ this(
+ Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ }
+
+ /**
+ * Creates {@code Lucene99PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * maxBlockSize} passed to block terms dictionary.
+ *
+ * @see
+ * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
+ */
+ public Lucene99PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ super("Lucene99");
+ Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ this.minTermBlockSize = minTermBlockSize;
+ this.maxTermBlockSize = maxTermBlockSize;
+ }
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
+ boolean success = false;
+ try {
+ FieldsConsumer ret =
+ new Lucene90BlockTreeTermsWriter(
+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+ success = true;
+ return ret;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(postingsWriter);
+ }
+ }
+ }
+
+ @Override
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
+ boolean success = false;
+ try {
+ FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);
+ success = true;
+ return ret;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(postingsReader);
+ }
+ }
+ }
+
+ /**
+ * Holds all state required for {@link Lucene99PostingsReader} to produce a {@link
+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
+ *
+ * @lucene.internal
+ */
+ public static final class IntBlockTermState extends BlockTermState {
+ /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */
+ public long docStartFP;
+
+ /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */
+ public long posStartFP;
+
+ /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */
+ public long payStartFP;
+
+ /**
+ * file offset for the start of the skip list, relative to docStartFP, if there are more than
+ * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1
+ */
+ public long skipOffset;
+
+ /**
+ * file offset for the last position in the last block, if there are more than {@link
+ * ForUtil#BLOCK_SIZE} positions; otherwise -1
+ *
+ * <p>One might think to use total term frequency to track how many positions are left to read
+ * as we decode the blocks, and decode the last block differently when num_left_positions &lt;
+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip
+ * blocks as the skipper will only tell us new position offset (start of block) and number of
+ * positions to skip for that block, without telling us how many positions it has skipped.
+ */
+ public long lastPosBlockOffset;
+
+ /**
+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
+ * totalTermFreq in this case.
+ */
+ public int singletonDocID;
+
+ /** Sole constructor. */
+ public IntBlockTermState() {
+ skipOffset = -1;
+ lastPosBlockOffset = -1;
+ singletonDocID = -1;
+ }
+
+ @Override
+ public IntBlockTermState clone() {
+ IntBlockTermState other = new IntBlockTermState();
+ other.copyFrom(this);
+ return other;
+ }
+
+ @Override
+ public void copyFrom(TermState _other) {
+ super.copyFrom(_other);
+ IntBlockTermState other = (IntBlockTermState) _other;
+ docStartFP = other.docStartFP;
+ posStartFP = other.posStartFP;
+ payStartFP = other.payStartFP;
+ lastPosBlockOffset = other.lastPosBlockOffset;
+ skipOffset = other.skipOffset;
+ singletonDocID = other.singletonDocID;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString()
+ + " docStartFP="
+ + docStartFP
+ + " posStartFP="
+ + posStartFP
+ + " payStartFP="
+ + payStartFP
+ + " lastPosBlockOffset="
+ + lastPosBlockOffset
+ + " singletonDocID="
+ + singletonDocID;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java
new file mode 100644
index 00000000000..77e50e23ce8
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java
@@ -0,0 +1,2087 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_START;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.PostingsReaderBase;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Impacts;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SlowImpactsEnum;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.
+ *
+ * @lucene.experimental
+ */
+public final class Lucene99PostingsReader extends PostingsReaderBase {
+
+ private final IndexInput docIn;
+ private final IndexInput posIn;
+ private final IndexInput payIn;
+
+ private final int version;
+
+ /** Sole constructor. */
+ public Lucene99PostingsReader(SegmentReadState state) throws IOException {
+ boolean success = false;
+ IndexInput docIn = null;
+ IndexInput posIn = null;
+ IndexInput payIn = null;
+
+ // NOTE: these data files are too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+
+ String docName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.DOC_EXTENSION);
+ try {
+ docIn = state.directory.openInput(docName, state.context);
+ version =
+ CodecUtil.checkIndexHeader(
+ docIn,
+ DOC_CODEC,
+ VERSION_START,
+ VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ CodecUtil.retrieveChecksum(docIn);
+
+ if (state.fieldInfos.hasProx()) {
+ String proxName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.POS_EXTENSION);
+ posIn = state.directory.openInput(proxName, state.context);
+ CodecUtil.checkIndexHeader(
+ posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(posIn);
+
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ String payName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name,
+ state.segmentSuffix,
+ Lucene99PostingsFormat.PAY_EXTENSION);
+ payIn = state.directory.openInput(payName, state.context);
+ CodecUtil.checkIndexHeader(
+ payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(payIn);
+ }
+ }
+
+ this.docIn = docIn;
+ this.posIn = posIn;
+ this.payIn = payIn;
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(docIn, posIn, payIn);
+ }
+ }
+ }
+
+ @Override
+ public void init(IndexInput termsIn, SegmentReadState state) throws IOException {
+ // Make sure we are talking to the matching postings writer
+ CodecUtil.checkIndexHeader(
+ termsIn,
+ TERMS_CODEC,
+ VERSION_START,
+ VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ final int indexBlockSize = termsIn.readVInt();
+ if (indexBlockSize != BLOCK_SIZE) {
+ throw new IllegalStateException(
+ "index-time BLOCK_SIZE ("
+ + indexBlockSize
+ + ") != read-time BLOCK_SIZE ("
+ + BLOCK_SIZE
+ + ")");
+ }
+ }
+
+ /** Read values that have been written using variable-length encoding instead of bit-packing. */
+ static void readVIntBlock(
+ IndexInput docIn, long[] docBuffer, long[] freqBuffer, int num, boolean indexHasFreq)
+ throws IOException {
+ if (indexHasFreq) {
+ for (int i = 0; i < num; i++) {
+ final int code = docIn.readVInt();
+ docBuffer[i] = code >>> 1;
+ if ((code & 1) != 0) {
+ freqBuffer[i] = 1;
+ } else {
+ freqBuffer[i] = docIn.readVInt();
+ }
+ }
+ } else {
+ for (int i = 0; i < num; i++) {
+ docBuffer[i] = docIn.readVInt();
+ }
+ }
+ }
+
+ static void prefixSum(long[] buffer, int count, long base) {
+ buffer[0] += base;
+ for (int i = 1; i < count; ++i) {
+ buffer[i] += buffer[i - 1];
+ }
+ }
+
+ static int findFirstGreater(long[] buffer, int target, int from) {
+ for (int i = from; i < BLOCK_SIZE; ++i) {
+ if (buffer[i] >= target) {
+ return i;
+ }
+ }
+ return BLOCK_SIZE;
+ }
+
+ @Override
+ public BlockTermState newTermState() {
+ return new IntBlockTermState();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(docIn, posIn, payIn);
+ }
+
+ @Override
+ public void decodeTerm(
+ DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
+ throws IOException {
+ final IntBlockTermState termState = (IntBlockTermState) _termState;
+ final boolean fieldHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ final boolean fieldHasOffsets =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ final boolean fieldHasPayloads = fieldInfo.hasPayloads();
+
+ if (absolute) {
+ termState.docStartFP = 0;
+ termState.posStartFP = 0;
+ termState.payStartFP = 0;
+ }
+
+ final long l = in.readVLong();
+ if ((l & 0x01) == 0) {
+ termState.docStartFP += l >>> 1;
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
+ } else {
+ termState.singletonDocID = -1;
+ }
+ } else {
+ assert absolute == false;
+ assert termState.singletonDocID != -1;
+ termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);
+ }
+
+ if (fieldHasPositions) {
+ termState.posStartFP += in.readVLong();
+ if (fieldHasOffsets || fieldHasPayloads) {
+ termState.payStartFP += in.readVLong();
+ }
+ if (termState.totalTermFreq > BLOCK_SIZE) {
+ termState.lastPosBlockOffset = in.readVLong();
+ } else {
+ termState.lastPosBlockOffset = -1;
+ }
+ }
+
+ if (termState.docFreq > BLOCK_SIZE) {
+ termState.skipOffset = in.readVLong();
+ } else {
+ termState.skipOffset = -1;
+ }
+ }
+
+ @Override
+ public PostingsEnum postings(
+ FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)
+ throws IOException {
+
+ boolean indexHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+
+ if (indexHasPositions == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
+ BlockDocsEnum docsEnum;
+ if (reuse instanceof BlockDocsEnum) {
+ docsEnum = (BlockDocsEnum) reuse;
+ if (!docsEnum.canReuse(docIn, fieldInfo)) {
+ docsEnum = new BlockDocsEnum(fieldInfo);
+ }
+ } else {
+ docsEnum = new BlockDocsEnum(fieldInfo);
+ }
+ return docsEnum.reset((IntBlockTermState) termState, flags);
+ } else {
+ EverythingEnum everythingEnum;
+ if (reuse instanceof EverythingEnum) {
+ everythingEnum = (EverythingEnum) reuse;
+ if (!everythingEnum.canReuse(docIn, fieldInfo)) {
+ everythingEnum = new EverythingEnum(fieldInfo);
+ }
+ } else {
+ everythingEnum = new EverythingEnum(fieldInfo);
+ }
+ return everythingEnum.reset((IntBlockTermState) termState, flags);
+ }
+ }
+
+ @Override
+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)
+ throws IOException {
+ if (state.docFreq <= BLOCK_SIZE) {
+ // no skip data
+ return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));
+ }
+
+ final boolean indexHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ final boolean indexHasOffsets =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ final boolean indexHasPayloads = fieldInfo.hasPayloads();
+
+ if (indexHasPositions == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
+ return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state);
+ }
+
+ if (indexHasPositions
+ && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)
+ && (indexHasOffsets == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)
+ && (indexHasPayloads == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {
+ return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);
+ }
+
+ return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);
+ }
+
+ final class BlockDocsEnum extends PostingsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+
+ private int docBufferUpto;
+
+ private Lucene99SkipReader skipper;
+ private boolean skipped;
+
+ final IndexInput startDocIn;
+
+ IndexInput docIn;
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)
+ private int blockUpto; // number of docs in or before the current block
+ private int doc; // doc we last read
+ private long accum; // accumulator for doc deltas
+
+ // Where this term's postings start in the .doc file:
+ private long docTermStartFP;
+
+ // Where this term's skip data starts (after
+ // docTermStartFP) in the .doc file (or -1 if there is
+ // no skip data for this term):
+ private long skipOffset;
+
+ // docID for next skip point, we won't use skipper if
+ // target docID is not larger than this
+ private int nextSkipDoc;
+
+ private boolean needsFreq; // true if the caller actually needs frequencies
+ // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block
+ // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer
+ // (needsFreq=false)
+ private boolean isFreqsRead;
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
+
+ public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
+ this.startDocIn = Lucene99PostingsReader.this.docIn;
+ this.docIn = null;
+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+ }
+
+ public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
+ return docIn == startDocIn
+ && indexHasFreq
+ == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)
+ && indexHasPos
+ == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ >= 0)
+ && indexHasPayloads == fieldInfo.hasPayloads();
+ }
+
+ public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {
+ docFreq = termState.docFreq;
+ totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;
+ docTermStartFP = termState.docStartFP;
+ skipOffset = termState.skipOffset;
+ singletonDocID = termState.singletonDocID;
+ if (docFreq > 1) {
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ docIn.seek(docTermStartFP);
+ }
+
+ doc = -1;
+ this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);
+ this.isFreqsRead = true;
+ if (indexHasFreq == false || needsFreq == false) {
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ freqBuffer[i] = 1;
+ }
+ }
+ accum = 0;
+ blockUpto = 0;
+ nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
+ docBufferUpto = BLOCK_SIZE;
+ skipped = false;
+ return this;
+ }
+
+ @Override
+ public int freq() throws IOException {
+ if (isFreqsRead == false) {
+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block
+ isFreqsRead = true;
+ }
+ return (int) freqBuffer[docBufferUpto - 1];
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public BytesRef getPayload() throws IOException {
+ return null;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn
+ // after it
+ if (isFreqsRead == false) {
+ pforUtil.skip(docIn);
+ isFreqsRead = true;
+ }
+
+ final int left = docFreq - blockUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
+
+ if (indexHasFreq) {
+ if (needsFreq) {
+ isFreqsRead = false;
+ } else {
+ pforUtil.skip(docIn); // skip over freqBuffer if we don't need them at all
+ }
+ }
+ blockUpto += BLOCK_SIZE;
+ } else if (docFreq == 1) {
+ docBuffer[0] = singletonDocID;
+ freqBuffer[0] = totalTermFreq;
+ docBuffer[1] = NO_MORE_DOCS;
+ blockUpto++;
+ } else {
+ // Read vInts:
+ readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);
+ prefixSum(docBuffer, left, accum);
+ docBuffer[left] = NO_MORE_DOCS;
+ blockUpto += left;
+ }
+ accum = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (docBufferUpto == BLOCK_SIZE) {
+ refillDocs(); // we don't need to load freqBuffer for now (will be loaded later if
+ // necessary)
+ }
+
+ doc = (int) docBuffer[docBufferUpto];
+ docBufferUpto++;
+ return doc;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ // current skip docID < docIDs generated from current buffer <= next skip docID
+ // we don't need to skip if target is buffered already
+ if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {
+
+ if (skipper == null) {
+ // Lazy init: first time this enum has ever been used for skipping
+ skipper =
+ new Lucene99SkipReader(
+ docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);
+ }
+
+ if (!skipped) {
+ assert skipOffset != -1;
+ // This is the first time this enum has skipped
+ // since reset() was called; load the skip data:
+ skipper.init(docTermStartFP + skipOffset, docTermStartFP, 0, 0, docFreq);
+ skipped = true;
+ }
+
+ // always plus one to fix the result, since skip position in Lucene99SkipReader
+ // is a little different from MultiLevelSkipListReader
+ final int newDocUpto = skipper.skipTo(target) + 1;
+
+ if (newDocUpto >= blockUpto) {
+ // Skipper moved
+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
+ blockUpto = newDocUpto;
+
+ // Force to read next block
+ docBufferUpto = BLOCK_SIZE;
+ accum = skipper.getDoc(); // actually, this is just lastSkipEntry
+ docIn.seek(skipper.getDocPointer()); // now point to the block we want to search
+ // even if freqBuffer were not read from the previous block, we will mark them as read,
+ // as we don't need to skip the previous block freqBuffer in refillDocs,
+ // as we have already positioned docIn where in needs to be.
+ isFreqsRead = true;
+ }
+ // next time we call advance, this is used to
+ // foresee whether skipper is necessary.
+ nextSkipDoc = skipper.getNextSkipDoc();
+ }
+ if (docBufferUpto == BLOCK_SIZE) {
+ refillDocs();
+ }
+
+ // Now scan... this is an inlined/pared down version
+ // of nextDoc():
+ long doc;
+ while (true) {
+ doc = docBuffer[docBufferUpto];
+
+ if (doc >= target) {
+ break;
+ }
+ ++docBufferUpto;
+ }
+
+ docBufferUpto++;
+ return this.doc = (int) doc;
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ // Also handles payloads + offsets
+ final class EverythingEnum extends PostingsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
+
+ private final long[] payloadLengthBuffer;
+ private final long[] offsetStartDeltaBuffer;
+ private final long[] offsetLengthBuffer;
+
+ private byte[] payloadBytes;
+ private int payloadByteUpto;
+ private int payloadLength;
+
+ private int lastStartOffset;
+ private int startOffset;
+ private int endOffset;
+
+ private int docBufferUpto;
+ private int posBufferUpto;
+
+ private Lucene99SkipReader skipper;
+ private boolean skipped;
+
+ final IndexInput startDocIn;
+
+ IndexInput docIn;
+ final IndexInput posIn;
+ final IndexInput payIn;
+ final BytesRef payload;
+
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // number of positions in this posting list
+ private int blockUpto; // number of docs in or before the current block
+ private int doc; // doc we last read
+ private long accum; // accumulator for doc deltas
+ private int freq; // freq we last read
+ private int position; // current position
+
+ // how many positions "behind" we are; nextPosition must
+ // skip these to "catch up":
+ private int posPendingCount;
+
+ // Lazy pos seek: if != -1 then we must seek to this FP
+ // before reading positions:
+ private long posPendingFP;
+
+ // Lazy pay seek: if != -1 then we must seek to this FP
+ // before reading payloads/offsets:
+ private long payPendingFP;
+
+ // Where this term's postings start in the .doc file:
+ private long docTermStartFP;
+
+ // Where this term's postings start in the .pos file:
+ private long posTermStartFP;
+
+ // Where this term's payloads/offsets start in the .pay
+ // file:
+ private long payTermStartFP;
+
+ // File pointer where the last (vInt encoded) pos delta
+ // block is. We need this to know whether to bulk
+ // decode vs vInt decode the block:
+ private long lastPosBlockFP;
+
+ // Where this term's skip data starts (after
+ // docTermStartFP) in the .doc file (or -1 if there is
+ // no skip data for this term):
+ private long skipOffset;
+
+ private int nextSkipDoc;
+
+ private boolean needsOffsets; // true if we actually need offsets
+ private boolean needsPayloads; // true if we actually need payloads
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
+
+ public EverythingEnum(FieldInfo fieldInfo) throws IOException {
+ indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+
+ this.startDocIn = Lucene99PostingsReader.this.docIn;
+ this.docIn = null;
+ this.posIn = Lucene99PostingsReader.this.posIn.clone();
+ if (indexHasOffsets || indexHasPayloads) {
+ this.payIn = Lucene99PostingsReader.this.payIn.clone();
+ } else {
+ this.payIn = null;
+ }
+ if (indexHasOffsets) {
+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];
+ offsetLengthBuffer = new long[BLOCK_SIZE];
+ } else {
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ startOffset = -1;
+ endOffset = -1;
+ }
+
+ if (indexHasPayloads) {
+ payloadLengthBuffer = new long[BLOCK_SIZE];
+ payloadBytes = new byte[128];
+ payload = new BytesRef();
+ } else {
+ payloadLengthBuffer = null;
+ payloadBytes = null;
+ payload = null;
+ }
+
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+ }
+
+ public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
+ return docIn == startDocIn
+ && indexHasOffsets
+ == (fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0)
+ && indexHasPayloads == fieldInfo.hasPayloads();
+ }
+
+ public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException {
+ docFreq = termState.docFreq;
+ docTermStartFP = termState.docStartFP;
+ posTermStartFP = termState.posStartFP;
+ payTermStartFP = termState.payStartFP;
+ skipOffset = termState.skipOffset;
+ totalTermFreq = termState.totalTermFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docFreq > 1) {
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ docIn.seek(docTermStartFP);
+ }
+ posPendingFP = posTermStartFP;
+ payPendingFP = payTermStartFP;
+ posPendingCount = 0;
+ if (termState.totalTermFreq < BLOCK_SIZE) {
+ lastPosBlockFP = posTermStartFP;
+ } else if (termState.totalTermFreq == BLOCK_SIZE) {
+ lastPosBlockFP = -1;
+ } else {
+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
+ }
+
+ this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
+ this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
+
+ doc = -1;
+ accum = 0;
+ blockUpto = 0;
+ if (docFreq > BLOCK_SIZE) {
+ nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
+ } else {
+ nextSkipDoc = NO_MORE_DOCS; // not enough docs for skipping
+ }
+ docBufferUpto = BLOCK_SIZE;
+ skipped = false;
+ return this;
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return freq;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ final int left = docFreq - blockUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
+ pforUtil.decode(docIn, freqBuffer);
+ blockUpto += BLOCK_SIZE;
+ } else if (docFreq == 1) {
+ docBuffer[0] = singletonDocID;
+ freqBuffer[0] = totalTermFreq;
+ docBuffer[1] = NO_MORE_DOCS;
+ blockUpto++;
+ } else {
+ readVIntBlock(docIn, docBuffer, freqBuffer, left, true);
+ prefixSum(docBuffer, left, accum);
+ docBuffer[left] = NO_MORE_DOCS;
+ blockUpto += left;
+ }
+ accum = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ private void refillPositions() throws IOException {
+ if (posIn.getFilePointer() == lastPosBlockFP) {
+ final int count = (int) (totalTermFreq % BLOCK_SIZE);
+ int payloadLength = 0;
+ int offsetLength = 0;
+ payloadByteUpto = 0;
+ for (int i = 0; i < count; i++) {
+ int code = posIn.readVInt();
+ if (indexHasPayloads) {
+ if ((code & 1) != 0) {
+ payloadLength = posIn.readVInt();
+ }
+ payloadLengthBuffer[i] = payloadLength;
+ posDeltaBuffer[i] = code >>> 1;
+ if (payloadLength != 0) {
+ if (payloadByteUpto + payloadLength > payloadBytes.length) {
+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
+ }
+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
+ payloadByteUpto += payloadLength;
+ }
+ } else {
+ posDeltaBuffer[i] = code;
+ }
+
+ if (indexHasOffsets) {
+ int deltaCode = posIn.readVInt();
+ if ((deltaCode & 1) != 0) {
+ offsetLength = posIn.readVInt();
+ }
+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;
+ offsetLengthBuffer[i] = offsetLength;
+ }
+ }
+ payloadByteUpto = 0;
+ } else {
+ pforUtil.decode(posIn, posDeltaBuffer);
+
+ if (indexHasPayloads) {
+ if (needsPayloads) {
+ pforUtil.decode(payIn, payloadLengthBuffer);
+ int numBytes = payIn.readVInt();
+
+ if (numBytes > payloadBytes.length) {
+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);
+ }
+ payIn.readBytes(payloadBytes, 0, numBytes);
+ } else {
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ pforUtil.skip(payIn); // skip over lengths
+ int numBytes = payIn.readVInt(); // read length of payloadBytes
+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
+ }
+ payloadByteUpto = 0;
+ }
+
+ if (indexHasOffsets) {
+ if (needsOffsets) {
+ pforUtil.decode(payIn, offsetStartDeltaBuffer);
+ pforUtil.decode(payIn, offsetLengthBuffer);
+ } else {
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ pforUtil.skip(payIn); // skip over starts
+ pforUtil.skip(payIn); // skip over lengths
+ }
+ }
+ }
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (docBufferUpto == BLOCK_SIZE) {
+ refillDocs();
+ }
+
+ doc = (int) docBuffer[docBufferUpto];
+ freq = (int) freqBuffer[docBufferUpto];
+ posPendingCount += freq;
+ docBufferUpto++;
+
+ position = 0;
+ lastStartOffset = 0;
+ return doc;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ if (skipper == null) {
+ // Lazy init: first time this enum has ever been used for skipping
+ skipper =
+ new Lucene99SkipReader(
+ docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);
+ }
+
+ if (!skipped) {
+ assert skipOffset != -1;
+ // This is the first time this enum has skipped
+ // since reset() was called; load the skip data:
+ skipper.init(
+ docTermStartFP + skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);
+ skipped = true;
+ }
+
+ final int newDocUpto = skipper.skipTo(target) + 1;
+
+ if (newDocUpto > blockUpto - BLOCK_SIZE + docBufferUpto) {
+ // Skipper moved
+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
+ blockUpto = newDocUpto;
+
+ // Force to read next block
+ docBufferUpto = BLOCK_SIZE;
+ accum = skipper.getDoc();
+ docIn.seek(skipper.getDocPointer());
+ posPendingFP = skipper.getPosPointer();
+ payPendingFP = skipper.getPayPointer();
+ posPendingCount = skipper.getPosBufferUpto();
+ lastStartOffset = 0; // new document
+ payloadByteUpto = skipper.getPayloadByteUpto();
+ }
+ nextSkipDoc = skipper.getNextSkipDoc();
+ }
+ if (docBufferUpto == BLOCK_SIZE) {
+ refillDocs();
+ }
+
+ // Now scan:
+ long doc;
+ while (true) {
+ doc = docBuffer[docBufferUpto];
+ freq = (int) freqBuffer[docBufferUpto];
+ posPendingCount += freq;
+ docBufferUpto++;
+
+ if (doc >= target) {
+ break;
+ }
+ }
+
+ position = 0;
+ lastStartOffset = 0;
+ return this.doc = (int) doc;
+ }
+
+ // TODO: in theory we could avoid loading frq block
+ // when not needed, ie, use skip data to load how far to
+ // seek the pos pointer ... instead of having to load frq
+ // blocks only to sum up how many positions to skip
+ private void skipPositions() throws IOException {
+ // Skip positions now:
+ int toSkip = posPendingCount - freq;
+ // if (DEBUG) {
+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
+ // }
+
+ final int leftInBlock = BLOCK_SIZE - posBufferUpto;
+ if (toSkip < leftInBlock) {
+ int end = posBufferUpto + toSkip;
+ while (posBufferUpto < end) {
+ if (indexHasPayloads) {
+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];
+ }
+ posBufferUpto++;
+ }
+ } else {
+ toSkip -= leftInBlock;
+ while (toSkip >= BLOCK_SIZE) {
+ assert posIn.getFilePointer() != lastPosBlockFP;
+ pforUtil.skip(posIn);
+
+ if (indexHasPayloads) {
+ // Skip payloadLength block:
+ pforUtil.skip(payIn);
+
+ // Skip payloadBytes block:
+ int numBytes = payIn.readVInt();
+ payIn.seek(payIn.getFilePointer() + numBytes);
+ }
+
+ if (indexHasOffsets) {
+ pforUtil.skip(payIn);
+ pforUtil.skip(payIn);
+ }
+ toSkip -= BLOCK_SIZE;
+ }
+ refillPositions();
+ payloadByteUpto = 0;
+ posBufferUpto = 0;
+ while (posBufferUpto < toSkip) {
+ if (indexHasPayloads) {
+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];
+ }
+ posBufferUpto++;
+ }
+ }
+
+ position = 0;
+ lastStartOffset = 0;
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ assert posPendingCount > 0;
+
+ if (posPendingFP != -1) {
+ posIn.seek(posPendingFP);
+ posPendingFP = -1;
+
+ if (payPendingFP != -1 && payIn != null) {
+ payIn.seek(payPendingFP);
+ payPendingFP = -1;
+ }
+
+ // Force buffer refill:
+ posBufferUpto = BLOCK_SIZE;
+ }
+
+ if (posPendingCount > freq) {
+ skipPositions();
+ posPendingCount = freq;
+ }
+
+ if (posBufferUpto == BLOCK_SIZE) {
+ refillPositions();
+ posBufferUpto = 0;
+ }
+ position += posDeltaBuffer[posBufferUpto];
+
+ if (indexHasPayloads) {
+ payloadLength = (int) payloadLengthBuffer[posBufferUpto];
+ payload.bytes = payloadBytes;
+ payload.offset = payloadByteUpto;
+ payload.length = payloadLength;
+ payloadByteUpto += payloadLength;
+ }
+
+ if (indexHasOffsets) {
+ startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];
+ endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];
+ lastStartOffset = startOffset;
+ }
+
+ posBufferUpto++;
+ posPendingCount--;
+ return position;
+ }
+
+ @Override
+ public int startOffset() {
+ return startOffset;
+ }
+
+ @Override
+ public int endOffset() {
+ return endOffset;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ if (payloadLength == 0) {
+ return null;
+ } else {
+ return payload;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ final class BlockImpactsDocsEnum extends ImpactsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+
+ private int docBufferUpto;
+
+ private final Lucene99ScoreSkipReader skipper;
+
+ final IndexInput docIn;
+
+ final boolean indexHasFreqs;
+
+ private int docFreq; // number of docs in this posting list
+ private int blockUpto; // number of documents in or before the current block
+ private int doc; // doc we last read
+ private long accum; // accumulator for doc deltas
+
+ private int nextSkipDoc = -1;
+
+ private long seekTo = -1;
+
+ // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block
+ // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer
+ // (needsFreq=false)
+ private boolean isFreqsRead;
+
+ public BlockImpactsDocsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
+ throws IOException {
+ indexHasFreqs = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ final boolean indexHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ final boolean indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ final boolean indexHasPayloads = fieldInfo.hasPayloads();
+
+ this.docIn = Lucene99PostingsReader.this.docIn.clone();
+
+ docFreq = termState.docFreq;
+ docIn.seek(termState.docStartFP);
+
+ doc = -1;
+ accum = 0;
+ blockUpto = 0;
+ docBufferUpto = BLOCK_SIZE;
+
+ skipper =
+ new Lucene99ScoreSkipReader(
+ docIn.clone(), MAX_SKIP_LEVELS, indexHasPositions, indexHasOffsets, indexHasPayloads);
+ skipper.init(
+ termState.docStartFP + termState.skipOffset,
+ termState.docStartFP,
+ termState.posStartFP,
+ termState.payStartFP,
+ docFreq);
+
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+ this.isFreqsRead = true;
+ if (indexHasFreqs == false) {
+ Arrays.fill(freqBuffer, 1L);
+ }
+ }
+
+ @Override
+ public int freq() throws IOException {
+ if (isFreqsRead == false) {
+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block
+ isFreqsRead = true;
+ }
+ return (int) freqBuffer[docBufferUpto - 1];
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn
+ // after it
+ if (isFreqsRead == false) {
+ pforUtil.skip(docIn);
+ isFreqsRead = true;
+ }
+
+ final int left = docFreq - blockUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
+ if (indexHasFreqs) {
+ isFreqsRead = false;
+ }
+ blockUpto += BLOCK_SIZE;
+ } else {
+ readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreqs);
+ prefixSum(docBuffer, left, accum);
+ docBuffer[left] = NO_MORE_DOCS;
+ blockUpto += left;
+ }
+ accum = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ @Override
+ public void advanceShallow(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ // always plus one to fix the result, since skip position in Lucene99SkipReader
+ // is a little different from MultiLevelSkipListReader
+ final int newDocUpto = skipper.skipTo(target) + 1;
+
+ if (newDocUpto >= blockUpto) {
+ // Skipper moved
+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
+ blockUpto = newDocUpto;
+
+ // Force to read next block
+ docBufferUpto = BLOCK_SIZE;
+ accum = skipper.getDoc();
+ seekTo = skipper.getDocPointer(); // delay the seek
+ }
+ // next time we call advance, this is used to
+ // foresee whether skipper is necessary.
+ nextSkipDoc = skipper.getNextSkipDoc();
+ }
+ assert nextSkipDoc >= target;
+ }
+
+ @Override
+ public Impacts getImpacts() throws IOException {
+ // nextDoc() doesn't advance skip lists, so it's important to do it here to make sure we're
+ // not returning impacts over a bigger range of doc IDs than necessary.
+ advanceShallow(doc);
+ return skipper.getImpacts();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (docBufferUpto == BLOCK_SIZE) {
+ if (seekTo >= 0) {
+ docIn.seek(seekTo);
+ isFreqsRead = true; // reset isFreqsRead
+ seekTo = -1;
+ }
+ refillDocs();
+ }
+ return this.doc = (int) docBuffer[docBufferUpto++];
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ advanceShallow(target);
+ }
+ if (docBufferUpto == BLOCK_SIZE) {
+ if (seekTo >= 0) {
+ docIn.seek(seekTo);
+ isFreqsRead = true; // reset isFreqsRead
+ seekTo = -1;
+ }
+ refillDocs();
+ }
+
+ int next = findFirstGreater(docBuffer, target, docBufferUpto);
+ this.doc = (int) docBuffer[next];
+ docBufferUpto = next + 1;
+ return doc;
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int startOffset() {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() {
+ return -1;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ return null;
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ final class BlockImpactsPostingsEnum extends ImpactsEnum {
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
+
+ private int docBufferUpto;
+ private int posBufferUpto;
+
+ private final Lucene99ScoreSkipReader skipper;
+
+ final IndexInput docIn;
+ final IndexInput posIn;
+
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // number of positions in this posting list
+ private int docUpto; // how many docs we've read
+ private int doc; // doc we last read
+ private long accum; // accumulator for doc deltas
+ private int freq; // freq we last read
+ private int position; // current position
+
+ // how many positions "behind" we are; nextPosition must
+ // skip these to "catch up":
+ private int posPendingCount;
+
+ // Lazy pos seek: if != -1 then we must seek to this FP
+ // before reading positions:
+ private long posPendingFP;
+
+ // Where this term's postings start in the .doc file:
+ private long docTermStartFP;
+
+ // Where this term's postings start in the .pos file:
+ private long posTermStartFP;
+
+ // Where this term's payloads/offsets start in the .pay
+ // file:
+ private long payTermStartFP;
+
+ // File pointer where the last (vInt encoded) pos delta
+ // block is. We need this to know whether to bulk
+ // decode vs vInt decode the block:
+ private long lastPosBlockFP;
+
+ private int nextSkipDoc = -1;
+
+ private long seekTo = -1;
+
+ public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
+ throws IOException {
+ indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+
+ this.docIn = Lucene99PostingsReader.this.docIn.clone();
+
+ this.posIn = Lucene99PostingsReader.this.posIn.clone();
+
+ docFreq = termState.docFreq;
+ docTermStartFP = termState.docStartFP;
+ posTermStartFP = termState.posStartFP;
+ payTermStartFP = termState.payStartFP;
+ totalTermFreq = termState.totalTermFreq;
+ docIn.seek(docTermStartFP);
+ posPendingFP = posTermStartFP;
+ posPendingCount = 0;
+ if (termState.totalTermFreq < BLOCK_SIZE) {
+ lastPosBlockFP = posTermStartFP;
+ } else if (termState.totalTermFreq == BLOCK_SIZE) {
+ lastPosBlockFP = -1;
+ } else {
+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
+ }
+
+ doc = -1;
+ accum = 0;
+ docUpto = 0;
+ docBufferUpto = BLOCK_SIZE;
+
+ skipper =
+ new Lucene99ScoreSkipReader(
+ docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);
+ skipper.init(
+ docTermStartFP + termState.skipOffset,
+ docTermStartFP,
+ posTermStartFP,
+ payTermStartFP,
+ docFreq);
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return freq;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ final int left = docFreq - docUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
+ pforUtil.decode(docIn, freqBuffer);
+ } else {
+ readVIntBlock(docIn, docBuffer, freqBuffer, left, true);
+ prefixSum(docBuffer, left, accum);
+ docBuffer[left] = NO_MORE_DOCS;
+ }
+ accum = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ }
+
+ private void refillPositions() throws IOException {
+ if (posIn.getFilePointer() == lastPosBlockFP) {
+ final int count = (int) (totalTermFreq % BLOCK_SIZE);
+ int payloadLength = 0;
+ for (int i = 0; i < count; i++) {
+ int code = posIn.readVInt();
+ if (indexHasPayloads) {
+ if ((code & 1) != 0) {
+ payloadLength = posIn.readVInt();
+ }
+ posDeltaBuffer[i] = code >>> 1;
+ if (payloadLength != 0) {
+ posIn.seek(posIn.getFilePointer() + payloadLength);
+ }
+ } else {
+ posDeltaBuffer[i] = code;
+ }
+ if (indexHasOffsets) {
+ if ((posIn.readVInt() & 1) != 0) {
+ // offset length changed
+ posIn.readVInt();
+ }
+ }
+ }
+ } else {
+ pforUtil.decode(posIn, posDeltaBuffer);
+ }
+ }
+
+ @Override
+ public void advanceShallow(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ // always plus one to fix the result, since skip position in Lucene99SkipReader
+ // is a little different from MultiLevelSkipListReader
+ final int newDocUpto = skipper.skipTo(target) + 1;
+
+ if (newDocUpto > docUpto) {
+ // Skipper moved
+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
+ docUpto = newDocUpto;
+
+ // Force to read next block
+ docBufferUpto = BLOCK_SIZE;
+ accum = skipper.getDoc();
+ posPendingFP = skipper.getPosPointer();
+ posPendingCount = skipper.getPosBufferUpto();
+ seekTo = skipper.getDocPointer(); // delay the seek
+ }
+ // next time we call advance, this is used to
+ // foresee whether skipper is necessary.
+ nextSkipDoc = skipper.getNextSkipDoc();
+ }
+ assert nextSkipDoc >= target;
+ }
+
+ @Override
+ public Impacts getImpacts() throws IOException {
+ advanceShallow(doc);
+ return skipper.getImpacts();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ advanceShallow(target);
+ }
+ if (docBufferUpto == BLOCK_SIZE) {
+ if (seekTo >= 0) {
+ docIn.seek(seekTo);
+ seekTo = -1;
+ }
+ refillDocs();
+ }
+
+ int next = findFirstGreater(docBuffer, target, docBufferUpto);
+ if (next == BLOCK_SIZE) {
+ return doc = NO_MORE_DOCS;
+ }
+ this.doc = (int) docBuffer[next];
+ this.freq = (int) freqBuffer[next];
+ for (int i = docBufferUpto; i <= next; ++i) {
+ posPendingCount += freqBuffer[i];
+ }
+ docUpto += next - docBufferUpto + 1;
+ docBufferUpto = next + 1;
+ position = 0;
+ return doc;
+ }
+
+ // TODO: in theory we could avoid loading frq block
+ // when not needed, ie, use skip data to load how far to
+ // seek the pos pointer ... instead of having to load frq
+ // blocks only to sum up how many positions to skip
+ private void skipPositions() throws IOException {
+ // Skip positions now:
+ int toSkip = posPendingCount - freq;
+
+ final int leftInBlock = BLOCK_SIZE - posBufferUpto;
+ if (toSkip < leftInBlock) {
+ posBufferUpto += toSkip;
+ } else {
+ toSkip -= leftInBlock;
+ while (toSkip >= BLOCK_SIZE) {
+ assert posIn.getFilePointer() != lastPosBlockFP;
+ pforUtil.skip(posIn);
+ toSkip -= BLOCK_SIZE;
+ }
+ refillPositions();
+ posBufferUpto = toSkip;
+ }
+
+ position = 0;
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ assert posPendingCount > 0;
+
+ if (posPendingFP != -1) {
+ posIn.seek(posPendingFP);
+ posPendingFP = -1;
+
+ // Force buffer refill:
+ posBufferUpto = BLOCK_SIZE;
+ }
+
+ if (posPendingCount > freq) {
+ skipPositions();
+ posPendingCount = freq;
+ }
+
+ if (posBufferUpto == BLOCK_SIZE) {
+ refillPositions();
+ posBufferUpto = 0;
+ }
+ position += posDeltaBuffer[posBufferUpto++];
+
+ posPendingCount--;
+ return position;
+ }
+
+ @Override
+ public int startOffset() {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() {
+ return -1;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ return null;
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ final class BlockImpactsEverythingEnum extends ImpactsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
+
+ private final long[] payloadLengthBuffer;
+ private final long[] offsetStartDeltaBuffer;
+ private final long[] offsetLengthBuffer;
+
+ private byte[] payloadBytes;
+ private int payloadByteUpto;
+ private int payloadLength;
+
+ private int lastStartOffset;
+ private int startOffset = -1;
+ private int endOffset = -1;
+
+ private int docBufferUpto;
+ private int posBufferUpto;
+
+ private final Lucene99ScoreSkipReader skipper;
+
+ final IndexInput docIn;
+ final IndexInput posIn;
+ final IndexInput payIn;
+ final BytesRef payload;
+
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // number of positions in this posting list
+ private int docUpto; // how many docs we've read
+ private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads
+ private int doc; // doc we last read
+ private long accum; // accumulator for doc deltas
+ private int position; // current position
+
+ // how many positions "behind" we are; nextPosition must
+ // skip these to "catch up":
+ private int posPendingCount;
+
+ // Lazy pos seek: if != -1 then we must seek to this FP
+ // before reading positions:
+ private long posPendingFP;
+
+ // Lazy pay seek: if != -1 then we must seek to this FP
+ // before reading payloads/offsets:
+ private long payPendingFP;
+
+ // Where this term's postings start in the .doc file:
+ private long docTermStartFP;
+
+ // Where this term's postings start in the .pos file:
+ private long posTermStartFP;
+
+ // Where this term's payloads/offsets start in the .pay
+ // file:
+ private long payTermStartFP;
+
+ // File pointer where the last (vInt encoded) pos delta
+ // block is. We need this to know whether to bulk
+ // decode vs vInt decode the block:
+ private long lastPosBlockFP;
+
+ private int nextSkipDoc = -1;
+
+ private final boolean needsPositions;
+ private final boolean needsOffsets; // true if we actually need offsets
+ private final boolean needsPayloads; // true if we actually need payloads
+
+ private boolean
+ isFreqsRead; // shows if freqBuffer for the current doc block are read into freqBuffer
+
+ private long seekTo = -1;
+
+ public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags)
+ throws IOException {
+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+
+ needsPositions = PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);
+ needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
+ needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
+
+ this.docIn = Lucene99PostingsReader.this.docIn.clone();
+
+ if (indexHasPos && needsPositions) {
+ this.posIn = Lucene99PostingsReader.this.posIn.clone();
+ } else {
+ this.posIn = null;
+ }
+
+ if ((indexHasOffsets && needsOffsets) || (indexHasPayloads && needsPayloads)) {
+ this.payIn = Lucene99PostingsReader.this.payIn.clone();
+ } else {
+ this.payIn = null;
+ }
+
+ if (indexHasOffsets) {
+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];
+ offsetLengthBuffer = new long[BLOCK_SIZE];
+ } else {
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ startOffset = -1;
+ endOffset = -1;
+ }
+
+ if (indexHasPayloads) {
+ payloadLengthBuffer = new long[BLOCK_SIZE];
+ payloadBytes = new byte[128];
+ payload = new BytesRef();
+ } else {
+ payloadLengthBuffer = null;
+ payloadBytes = null;
+ payload = null;
+ }
+
+ docFreq = termState.docFreq;
+ docTermStartFP = termState.docStartFP;
+ posTermStartFP = termState.posStartFP;
+ payTermStartFP = termState.payStartFP;
+ totalTermFreq = termState.totalTermFreq;
+ docIn.seek(docTermStartFP);
+ posPendingFP = posTermStartFP;
+ payPendingFP = payTermStartFP;
+ posPendingCount = 0;
+ if (termState.totalTermFreq < BLOCK_SIZE) {
+ lastPosBlockFP = posTermStartFP;
+ } else if (termState.totalTermFreq == BLOCK_SIZE) {
+ lastPosBlockFP = -1;
+ } else {
+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
+ }
+
+ doc = -1;
+ accum = 0;
+ docUpto = 0;
+ posDocUpTo = 0;
+ isFreqsRead = true;
+ docBufferUpto = BLOCK_SIZE;
+
+ skipper =
+ new Lucene99ScoreSkipReader(
+ docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);
+ skipper.init(
+ docTermStartFP + termState.skipOffset,
+ docTermStartFP,
+ posTermStartFP,
+ payTermStartFP,
+ docFreq);
+
+ if (indexHasFreq == false) {
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ freqBuffer[i] = 1;
+ }
+ }
+ }
+
+ @Override
+ public int freq() throws IOException {
+ if (indexHasFreq && (isFreqsRead == false)) {
+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block
+ isFreqsRead = true;
+ }
+ return (int) freqBuffer[docBufferUpto - 1];
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ if (indexHasFreq) {
+ if (isFreqsRead == false) { // previous freq block was not read
+ // check if we need to load the previous freq block to catch up on positions or we can
+ // skip it
+ if (indexHasPos && needsPositions && (posDocUpTo < docUpto)) {
+ pforUtil.decode(docIn, freqBuffer); // load the previous freq block
+ } else {
+ pforUtil.skip(docIn); // skip it
+ }
+ isFreqsRead = true;
+ }
+ if (indexHasPos && needsPositions) {
+ while (posDocUpTo
+ < docUpto) { // catch on positions, bring posPendingCount upto the current doc
+ posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];
+ posDocUpTo++;
+ }
+ }
+ }
+
+ final int left = docFreq - docUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
+ if (indexHasFreq) {
+ isFreqsRead =
+ false; // freq block will be loaded lazily when necessary, we don't load it here
+ }
+ } else {
+ readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);
+ prefixSum(docBuffer, left, accum);
+ docBuffer[left] = NO_MORE_DOCS;
+ }
+ accum = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ }
+
+ private void refillPositions() throws IOException {
+ if (posIn.getFilePointer() == lastPosBlockFP) {
+ final int count = (int) (totalTermFreq % BLOCK_SIZE);
+ int payloadLength = 0;
+ int offsetLength = 0;
+ payloadByteUpto = 0;
+ for (int i = 0; i < count; i++) {
+ int code = posIn.readVInt();
+ if (indexHasPayloads) {
+ if ((code & 1) != 0) {
+ payloadLength = posIn.readVInt();
+ }
+ payloadLengthBuffer[i] = payloadLength;
+ posDeltaBuffer[i] = code >>> 1;
+ if (payloadLength != 0) {
+ if (payloadByteUpto + payloadLength > payloadBytes.length) {
+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
+ }
+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
+ payloadByteUpto += payloadLength;
+ }
+ } else {
+ posDeltaBuffer[i] = code;
+ }
+
+ if (indexHasOffsets) {
+ int deltaCode = posIn.readVInt();
+ if ((deltaCode & 1) != 0) {
+ offsetLength = posIn.readVInt();
+ }
+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;
+ offsetLengthBuffer[i] = offsetLength;
+ }
+ }
+ payloadByteUpto = 0;
+ } else {
+ pforUtil.decode(posIn, posDeltaBuffer);
+
+ if (indexHasPayloads && payIn != null) {
+ if (needsPayloads) {
+ pforUtil.decode(payIn, payloadLengthBuffer);
+ int numBytes = payIn.readVInt();
+
+ if (numBytes > payloadBytes.length) {
+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);
+ }
+ payIn.readBytes(payloadBytes, 0, numBytes);
+ } else {
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ pforUtil.skip(payIn); // skip over lengths
+ int numBytes = payIn.readVInt(); // read length of payloadBytes
+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
+ }
+ payloadByteUpto = 0;
+ }
+
+ if (indexHasOffsets && payIn != null) {
+ if (needsOffsets) {
+ pforUtil.decode(payIn, offsetStartDeltaBuffer);
+ pforUtil.decode(payIn, offsetLengthBuffer);
+ } else {
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ pforUtil.skip(payIn); // skip over starts
+ pforUtil.skip(payIn); // skip over lengths
+ }
+ }
+ }
+ }
+
+ @Override
+ public void advanceShallow(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ // always plus one to fix the result, since skip position in Lucene99SkipReader
+ // is a little different from MultiLevelSkipListReader
+ final int newDocUpto = skipper.skipTo(target) + 1;
+
+ if (newDocUpto > docUpto) {
+ // Skipper moved
+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
+ docUpto = newDocUpto;
+ posDocUpTo = docUpto;
+
+ // Force to read next block
+ docBufferUpto = BLOCK_SIZE;
+ accum = skipper.getDoc();
+ posPendingFP = skipper.getPosPointer();
+ payPendingFP = skipper.getPayPointer();
+ posPendingCount = skipper.getPosBufferUpto();
+ lastStartOffset = 0; // new document
+ payloadByteUpto = skipper.getPayloadByteUpto(); // actually, this is just lastSkipEntry
+ seekTo = skipper.getDocPointer(); // delay the seek
+ }
+ // next time we call advance, this is used to
+ // foresee whether skipper is necessary.
+ nextSkipDoc = skipper.getNextSkipDoc();
+ }
+ assert nextSkipDoc >= target;
+ }
+
+ @Override
+ public Impacts getImpacts() throws IOException {
+ advanceShallow(doc);
+ return skipper.getImpacts();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > nextSkipDoc) {
+ advanceShallow(target);
+ }
+ if (docBufferUpto == BLOCK_SIZE) {
+ if (seekTo >= 0) {
+ docIn.seek(seekTo);
+ seekTo = -1;
+ isFreqsRead = true; // reset isFreqsRead
+ }
+ refillDocs();
+ }
+
+ // Now scan:
+ long doc;
+ while (true) {
+ doc = docBuffer[docBufferUpto];
+ docBufferUpto++;
+ docUpto++;
+
+ if (doc >= target) {
+ break;
+ }
+
+ if (docBufferUpto == BLOCK_SIZE) {
+ return this.doc = NO_MORE_DOCS;
+ }
+ }
+ position = 0;
+ lastStartOffset = 0;
+
+ return this.doc = (int) doc;
+ }
+
+ // TODO: in theory we could avoid loading frq block
+ // when not needed, ie, use skip data to load how far to
+ // seek the pos pointer ... instead of having to load frq
+ // blocks only to sum up how many positions to skip
+ private void skipPositions() throws IOException {
+ // Skip positions now:
+ int toSkip = posPendingCount - (int) freqBuffer[docBufferUpto - 1];
+ // if (DEBUG) {
+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
+ // }
+
+ final int leftInBlock = BLOCK_SIZE - posBufferUpto;
+ if (toSkip < leftInBlock) {
+ int end = posBufferUpto + toSkip;
+ while (posBufferUpto < end) {
+ if (indexHasPayloads) {
+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];
+ }
+ posBufferUpto++;
+ }
+ } else {
+ toSkip -= leftInBlock;
+ while (toSkip >= BLOCK_SIZE) {
+ assert posIn.getFilePointer() != lastPosBlockFP;
+ pforUtil.skip(posIn);
+
+ if (indexHasPayloads && payIn != null) {
+ // Skip payloadLength block:
+ pforUtil.skip(payIn);
+
+ // Skip payloadBytes block:
+ int numBytes = payIn.readVInt();
+ payIn.seek(payIn.getFilePointer() + numBytes);
+ }
+
+ if (indexHasOffsets && payIn != null) {
+ pforUtil.skip(payIn);
+ pforUtil.skip(payIn);
+ }
+ toSkip -= BLOCK_SIZE;
+ }
+ refillPositions();
+ payloadByteUpto = 0;
+ posBufferUpto = 0;
+ while (posBufferUpto < toSkip) {
+ if (indexHasPayloads) {
+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];
+ }
+ posBufferUpto++;
+ }
+ }
+
+ position = 0;
+ lastStartOffset = 0;
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ if (indexHasPos == false || needsPositions == false) {
+ return -1;
+ }
+
+ if (isFreqsRead == false) {
+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this docs block
+ isFreqsRead = true;
+ }
+ while (posDocUpTo < docUpto) { // bring posPendingCount upto the current doc
+ posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];
+ posDocUpTo++;
+ }
+
+ assert posPendingCount > 0;
+
+ if (posPendingFP != -1) {
+ posIn.seek(posPendingFP);
+ posPendingFP = -1;
+
+ if (payPendingFP != -1 && payIn != null) {
+ payIn.seek(payPendingFP);
+ payPendingFP = -1;
+ }
+
+ // Force buffer refill:
+ posBufferUpto = BLOCK_SIZE;
+ }
+
+ if (posPendingCount > freqBuffer[docBufferUpto - 1]) {
+ skipPositions();
+ posPendingCount = (int) freqBuffer[docBufferUpto - 1];
+ }
+
+ if (posBufferUpto == BLOCK_SIZE) {
+ refillPositions();
+ posBufferUpto = 0;
+ }
+ position += posDeltaBuffer[posBufferUpto];
+
+ if (indexHasPayloads) {
+ payloadLength = (int) payloadLengthBuffer[posBufferUpto];
+ payload.bytes = payloadBytes;
+ payload.offset = payloadByteUpto;
+ payload.length = payloadLength;
+ payloadByteUpto += payloadLength;
+ }
+
+ if (indexHasOffsets && needsOffsets) {
+ startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];
+ endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];
+ lastStartOffset = startOffset;
+ }
+
+ posBufferUpto++;
+ posPendingCount--;
+ return position;
+ }
+
+ @Override
+ public int startOffset() {
+ return startOffset;
+ }
+
+ @Override
+ public int endOffset() {
+ return endOffset;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ if (payloadLength == 0) {
+ return null;
+ } else {
+ return payload;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (docIn != null) {
+ CodecUtil.checksumEntireFile(docIn);
+ }
+ if (posIn != null) {
+ CodecUtil.checksumEntireFile(posIn);
+ }
+ if (payIn != null) {
+ CodecUtil.checksumEntireFile(payIn);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName()
+ + "(positions="
+ + (posIn != null)
+ + ",payloads="
+ + (payIn != null)
+ + ")";
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java
new file mode 100644
index 00000000000..34b4c9c62a3
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java
@@ -0,0 +1,539 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
+import org.apache.lucene.codecs.PushPostingsWriterBase;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.
+ *
+ * <p>Postings list for each term will be stored separately.
+ *
+ * @see Lucene99SkipWriter for details about skipping setting and postings layout.
+ * @lucene.experimental
+ */
+public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
+
+ IndexOutput docOut;
+ IndexOutput posOut;
+ IndexOutput payOut;
+
+ static final IntBlockTermState emptyState = new IntBlockTermState();
+ IntBlockTermState lastState;
+
+ // Holds starting file pointers for current term:
+ private long docStartFP;
+ private long posStartFP;
+ private long payStartFP;
+
+ final long[] docDeltaBuffer;
+ final long[] freqBuffer;
+ private int docBufferUpto;
+
+ final long[] posDeltaBuffer;
+ final long[] payloadLengthBuffer;
+ final long[] offsetStartDeltaBuffer;
+ final long[] offsetLengthBuffer;
+ private int posBufferUpto;
+
+ private byte[] payloadBytes;
+ private int payloadByteUpto;
+
+ private int lastBlockDocID;
+ private long lastBlockPosFP;
+ private long lastBlockPayFP;
+ private int lastBlockPosBufferUpto;
+ private int lastBlockPayloadByteUpto;
+
+ private int lastDocID;
+ private int lastPosition;
+ private int lastStartOffset;
+ private int docCount;
+
+ private final PForUtil pforUtil;
+ private final ForDeltaUtil forDeltaUtil;
+ private final Lucene99SkipWriter skipWriter;
+
+ private boolean fieldHasNorms;
+ private NumericDocValues norms;
+ private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =
+ new CompetitiveImpactAccumulator();
+
+ /** Creates a postings writer */
+ public Lucene99PostingsWriter(SegmentWriteState state) throws IOException {
+
+ String docFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.DOC_EXTENSION);
+ docOut = state.directory.createOutput(docFileName, state.context);
+ IndexOutput posOut = null;
+ IndexOutput payOut = null;
+ boolean success = false;
+ try {
+ CodecUtil.writeIndexHeader(
+ docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ final ForUtil forUtil = new ForUtil();
+ forDeltaUtil = new ForDeltaUtil(forUtil);
+ pforUtil = new PForUtil(forUtil);
+ if (state.fieldInfos.hasProx()) {
+ posDeltaBuffer = new long[BLOCK_SIZE];
+ String posFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.POS_EXTENSION);
+ posOut = state.directory.createOutput(posFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+
+ if (state.fieldInfos.hasPayloads()) {
+ payloadBytes = new byte[128];
+ payloadLengthBuffer = new long[BLOCK_SIZE];
+ } else {
+ payloadBytes = null;
+ payloadLengthBuffer = null;
+ }
+
+ if (state.fieldInfos.hasOffsets()) {
+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];
+ offsetLengthBuffer = new long[BLOCK_SIZE];
+ } else {
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ }
+
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ String payFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name,
+ state.segmentSuffix,
+ Lucene99PostingsFormat.PAY_EXTENSION);
+ payOut = state.directory.createOutput(payFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ }
+ } else {
+ posDeltaBuffer = null;
+ payloadLengthBuffer = null;
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ payloadBytes = null;
+ }
+ this.payOut = payOut;
+ this.posOut = posOut;
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
+ }
+ }
+
+ docDeltaBuffer = new long[BLOCK_SIZE];
+ freqBuffer = new long[BLOCK_SIZE];
+
+ // TODO: should we try skipping every 2/4 blocks...?
+ skipWriter =
+ new Lucene99SkipWriter(
+ MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);
+ }
+
+ @Override
+ public IntBlockTermState newTermState() {
+ return new IntBlockTermState();
+ }
+
+ @Override
+ public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
+ CodecUtil.writeIndexHeader(
+ termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ termsOut.writeVInt(BLOCK_SIZE);
+ }
+
+ @Override
+ public void setField(FieldInfo fieldInfo) {
+ super.setField(fieldInfo);
+ skipWriter.setField(writePositions, writeOffsets, writePayloads);
+ lastState = emptyState;
+ fieldHasNorms = fieldInfo.hasNorms();
+ }
+
+ @Override
+ public void startTerm(NumericDocValues norms) {
+ docStartFP = docOut.getFilePointer();
+ if (writePositions) {
+ posStartFP = posOut.getFilePointer();
+ if (writePayloads || writeOffsets) {
+ payStartFP = payOut.getFilePointer();
+ }
+ }
+ lastDocID = 0;
+ lastBlockDocID = -1;
+ skipWriter.resetSkip();
+ this.norms = norms;
+ competitiveFreqNormAccumulator.clear();
+ }
+
+ @Override
+ public void startDoc(int docID, int termDocFreq) throws IOException {
+ // Have collected a block of docs, and get a new doc.
+ // Should write skip data as well as postings list for
+ // current block.
+ if (lastBlockDocID != -1 && docBufferUpto == 0) {
+ skipWriter.bufferSkip(
+ lastBlockDocID,
+ competitiveFreqNormAccumulator,
+ docCount,
+ lastBlockPosFP,
+ lastBlockPayFP,
+ lastBlockPosBufferUpto,
+ lastBlockPayloadByteUpto);
+ competitiveFreqNormAccumulator.clear();
+ }
+
+ final int docDelta = docID - lastDocID;
+
+ if (docID < 0 || (docCount > 0 && docDelta <= 0)) {
+ throw new CorruptIndexException(
+ "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);
+ }
+
+ docDeltaBuffer[docBufferUpto] = docDelta;
+ if (writeFreqs) {
+ freqBuffer[docBufferUpto] = termDocFreq;
+ }
+
+ docBufferUpto++;
+ docCount++;
+
+ if (docBufferUpto == BLOCK_SIZE) {
+ forDeltaUtil.encodeDeltas(docDeltaBuffer, docOut);
+ if (writeFreqs) {
+ pforUtil.encode(freqBuffer, docOut);
+ }
+ // NOTE: don't set docBufferUpto back to 0 here;
+ // finishDoc will do so (because it needs to see that
+ // the block was filled so it can save skip data)
+ }
+
+ lastDocID = docID;
+ lastPosition = 0;
+ lastStartOffset = 0;
+
+ long norm;
+ if (fieldHasNorms) {
+ boolean found = norms.advanceExact(docID);
+ if (found == false) {
+ // This can happen if indexing hits a problem after adding a doc to the
+ // postings but before buffering the norm. Such documents are written
+ // deleted and will go away on the first merge.
+ norm = 1L;
+ } else {
+ norm = norms.longValue();
+ assert norm != 0 : docID;
+ }
+ } else {
+ norm = 1L;
+ }
+
+ competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);
+ }
+
+ @Override
+ public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)
+ throws IOException {
+ if (position > IndexWriter.MAX_POSITION) {
+ throw new CorruptIndexException(
+ "position="
+ + position
+ + " is too large (> IndexWriter.MAX_POSITION="
+ + IndexWriter.MAX_POSITION
+ + ")",
+ docOut);
+ }
+ if (position < 0) {
+ throw new CorruptIndexException("position=" + position + " is < 0", docOut);
+ }
+ posDeltaBuffer[posBufferUpto] = position - lastPosition;
+ if (writePayloads) {
+ if (payload == null || payload.length == 0) {
+ // no payload
+ payloadLengthBuffer[posBufferUpto] = 0;
+ } else {
+ payloadLengthBuffer[posBufferUpto] = payload.length;
+ if (payloadByteUpto + payload.length > payloadBytes.length) {
+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);
+ }
+ System.arraycopy(
+ payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);
+ payloadByteUpto += payload.length;
+ }
+ }
+
+ if (writeOffsets) {
+ assert startOffset >= lastStartOffset;
+ assert endOffset >= startOffset;
+ offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
+ offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
+ lastStartOffset = startOffset;
+ }
+
+ posBufferUpto++;
+ lastPosition = position;
+ if (posBufferUpto == BLOCK_SIZE) {
+ pforUtil.encode(posDeltaBuffer, posOut);
+
+ if (writePayloads) {
+ pforUtil.encode(payloadLengthBuffer, payOut);
+ payOut.writeVInt(payloadByteUpto);
+ payOut.writeBytes(payloadBytes, 0, payloadByteUpto);
+ payloadByteUpto = 0;
+ }
+ if (writeOffsets) {
+ pforUtil.encode(offsetStartDeltaBuffer, payOut);
+ pforUtil.encode(offsetLengthBuffer, payOut);
+ }
+ posBufferUpto = 0;
+ }
+ }
+
+ @Override
+ public void finishDoc() throws IOException {
+ // Since we don't know df for current term, we had to buffer
+ // those skip data for each block, and when a new doc comes,
+ // write them to skip file.
+ if (docBufferUpto == BLOCK_SIZE) {
+ lastBlockDocID = lastDocID;
+ if (posOut != null) {
+ if (payOut != null) {
+ lastBlockPayFP = payOut.getFilePointer();
+ }
+ lastBlockPosFP = posOut.getFilePointer();
+ lastBlockPosBufferUpto = posBufferUpto;
+ lastBlockPayloadByteUpto = payloadByteUpto;
+ }
+ docBufferUpto = 0;
+ }
+ }
+
+ /** Called when we are done adding docs to this term */
+ @Override
+ public void finishTerm(BlockTermState _state) throws IOException {
+ IntBlockTermState state = (IntBlockTermState) _state;
+ assert state.docFreq > 0;
+
+ // TODO: wasteful we are counting this (counting # docs
+ // for this term) in two places?
+ assert state.docFreq == docCount : state.docFreq + " vs " + docCount;
+
+ // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to
+ // it.
+ final int singletonDocID;
+ if (state.docFreq == 1) {
+ // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
+ singletonDocID = (int) docDeltaBuffer[0];
+ } else {
+ singletonDocID = -1;
+ // vInt encode the remaining doc deltas and freqs:
+ for (int i = 0; i < docBufferUpto; i++) {
+ final int docDelta = (int) docDeltaBuffer[i];
+ final int freq = (int) freqBuffer[i];
+ if (!writeFreqs) {
+ docOut.writeVInt(docDelta);
+ } else if (freq == 1) {
+ docOut.writeVInt((docDelta << 1) | 1);
+ } else {
+ docOut.writeVInt(docDelta << 1);
+ docOut.writeVInt(freq);
+ }
+ }
+ }
+
+ final long lastPosBlockOffset;
+
+ if (writePositions) {
+ // totalTermFreq is just total number of positions(or payloads, or offsets)
+ // associated with current term.
+ assert state.totalTermFreq != -1;
+ if (state.totalTermFreq > BLOCK_SIZE) {
+ // record file offset for last pos in last block
+ lastPosBlockOffset = posOut.getFilePointer() - posStartFP;
+ } else {
+ lastPosBlockOffset = -1;
+ }
+ if (posBufferUpto > 0) {
+ // TODO: should we send offsets/payloads to
+ // .pay...? seems wasteful (have to store extra
+ // vLong for low (< BLOCK_SIZE) DF terms = vast vast
+ // majority)
+
+ // vInt encode the remaining positions/payloads/offsets:
+ int lastPayloadLength = -1; // force first payload length to be written
+ int lastOffsetLength = -1; // force first offset length to be written
+ int payloadBytesReadUpto = 0;
+ for (int i = 0; i < posBufferUpto; i++) {
+ final int posDelta = (int) posDeltaBuffer[i];
+ if (writePayloads) {
+ final int payloadLength = (int) payloadLengthBuffer[i];
+ if (payloadLength != lastPayloadLength) {
+ lastPayloadLength = payloadLength;
+ posOut.writeVInt((posDelta << 1) | 1);
+ posOut.writeVInt(payloadLength);
+ } else {
+ posOut.writeVInt(posDelta << 1);
+ }
+
+ if (payloadLength != 0) {
+ posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);
+ payloadBytesReadUpto += payloadLength;
+ }
+ } else {
+ posOut.writeVInt(posDelta);
+ }
+
+ if (writeOffsets) {
+ int delta = (int) offsetStartDeltaBuffer[i];
+ int length = (int) offsetLengthBuffer[i];
+ if (length == lastOffsetLength) {
+ posOut.writeVInt(delta << 1);
+ } else {
+ posOut.writeVInt(delta << 1 | 1);
+ posOut.writeVInt(length);
+ lastOffsetLength = length;
+ }
+ }
+ }
+
+ if (writePayloads) {
+ assert payloadBytesReadUpto == payloadByteUpto;
+ payloadByteUpto = 0;
+ }
+ }
+ } else {
+ lastPosBlockOffset = -1;
+ }
+
+ long skipOffset;
+ if (docCount > BLOCK_SIZE) {
+ skipOffset = skipWriter.writeSkip(docOut) - docStartFP;
+ } else {
+ skipOffset = -1;
+ }
+
+ state.docStartFP = docStartFP;
+ state.posStartFP = posStartFP;
+ state.payStartFP = payStartFP;
+ state.singletonDocID = singletonDocID;
+ state.skipOffset = skipOffset;
+ state.lastPosBlockOffset = lastPosBlockOffset;
+ docBufferUpto = 0;
+ posBufferUpto = 0;
+ lastDocID = 0;
+ docCount = 0;
+ }
+
+ @Override
+ public void encodeTerm(
+ DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)
+ throws IOException {
+ IntBlockTermState state = (IntBlockTermState) _state;
+ if (absolute) {
+ lastState = emptyState;
+ assert lastState.docStartFP == 0;
+ }
+
+ if (lastState.singletonDocID != -1
+ && state.singletonDocID != -1
+ && state.docStartFP == lastState.docStartFP) {
+ // With runs of rare values such as ID fields, the increment of pointers in the docs file is
+ // often 0.
+ // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we
+ // encode the delta
+ // between consecutive doc IDs to save space.
+ final long delta = (long) state.singletonDocID - lastState.singletonDocID;
+ out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);
+ } else {
+ out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);
+ if (state.singletonDocID != -1) {
+ out.writeVInt(state.singletonDocID);
+ }
+ }
+
+ if (writePositions) {
+ out.writeVLong(state.posStartFP - lastState.posStartFP);
+ if (writePayloads || writeOffsets) {
+ out.writeVLong(state.payStartFP - lastState.payStartFP);
+ }
+ }
+ if (writePositions) {
+ if (state.lastPosBlockOffset != -1) {
+ out.writeVLong(state.lastPosBlockOffset);
+ }
+ }
+ if (state.skipOffset != -1) {
+ out.writeVLong(state.skipOffset);
+ }
+ lastState = state;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // TODO: add a finish() at least to PushBase? DV too...?
+ boolean success = false;
+ try {
+ if (docOut != null) {
+ CodecUtil.writeFooter(docOut);
+ }
+ if (posOut != null) {
+ CodecUtil.writeFooter(posOut);
+ }
+ if (payOut != null) {
+ CodecUtil.writeFooter(payOut);
+ }
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(docOut, posOut, payOut);
+ } else {
+ IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
+ }
+ docOut = posOut = payOut = null;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java
new file mode 100644
index 00000000000..f2882c085c4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import java.util.AbstractList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.RandomAccess;
+import org.apache.lucene.index.Impact;
+import org.apache.lucene.index.Impacts;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+
+final class Lucene99ScoreSkipReader extends Lucene99SkipReader {
+
+ private final byte[][] impactData;
+ private final int[] impactDataLength;
+ private final ByteArrayDataInput badi = new ByteArrayDataInput();
+ private final Impacts impacts;
+ private int numLevels = 1;
+ private final MutableImpactList[] perLevelImpacts;
+
+ public Lucene99ScoreSkipReader(
+ IndexInput skipStream,
+ int maxSkipLevels,
+ boolean hasPos,
+ boolean hasOffsets,
+ boolean hasPayloads) {
+ super(skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);
+ this.impactData = new byte[maxSkipLevels][];
+ Arrays.fill(impactData, new byte[0]);
+ this.impactDataLength = new int[maxSkipLevels];
+ this.perLevelImpacts = new MutableImpactList[maxSkipLevels];
+ for (int i = 0; i < perLevelImpacts.length; ++i) {
+ perLevelImpacts[i] = new MutableImpactList();
+ }
+ impacts =
+ new Impacts() {
+
+ @Override
+ public int numLevels() {
+ return numLevels;
+ }
+
+ @Override
+ public int getDocIdUpTo(int level) {
+ return skipDoc[level];
+ }
+
+ @Override
+ public List<Impact> getImpacts(int level) {
+ assert level < numLevels;
+ if (impactDataLength[level] > 0) {
+ badi.reset(impactData[level], 0, impactDataLength[level]);
+ perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);
+ impactDataLength[level] = 0;
+ }
+ return perLevelImpacts[level];
+ }
+ };
+ }
+
+ @Override
+ public int skipTo(int target) throws IOException {
+ int result = super.skipTo(target);
+ if (numberOfSkipLevels > 0) {
+ numLevels = numberOfSkipLevels;
+ } else {
+ // End of postings don't have skip data anymore, so we fill with dummy data
+ // like SlowImpactsEnum.
+ numLevels = 1;
+ perLevelImpacts[0].length = 1;
+ perLevelImpacts[0].impacts[0].freq = Integer.MAX_VALUE;
+ perLevelImpacts[0].impacts[0].norm = 1L;
+ impactDataLength[0] = 0;
+ }
+ return result;
+ }
+
+ Impacts getImpacts() {
+ return impacts;
+ }
+
+ @Override
+ protected void readImpacts(int level, IndexInput skipStream) throws IOException {
+ int length = skipStream.readVInt();
+ if (impactData[level].length < length) {
+ impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
+ }
+ skipStream.readBytes(impactData[level], 0, length);
+ impactDataLength[level] = length;
+ }
+
+ static MutableImpactList readImpacts(ByteArrayDataInput in, MutableImpactList reuse) {
+ int maxNumImpacts = in.length(); // at most one impact per byte
+ if (reuse.impacts.length < maxNumImpacts) {
+ int oldLength = reuse.impacts.length;
+ reuse.impacts = ArrayUtil.grow(reuse.impacts, maxNumImpacts);
+ for (int i = oldLength; i < reuse.impacts.length; ++i) {
+ reuse.impacts[i] = new Impact(Integer.MAX_VALUE, 1L);
+ }
+ }
+
+ int freq = 0;
+ long norm = 0;
+ int length = 0;
+ while (in.getPosition() < in.length()) {
+ int freqDelta = in.readVInt();
+ if ((freqDelta & 0x01) != 0) {
+ freq += 1 + (freqDelta >>> 1);
+ try {
+ norm += 1 + in.readZLong();
+ } catch (IOException e) {
+ throw new RuntimeException(e); // cannot happen on a BADI
+ }
+ } else {
+ freq += 1 + (freqDelta >>> 1);
+ norm++;
+ }
+ Impact impact = reuse.impacts[length];
+ impact.freq = freq;
+ impact.norm = norm;
+ length++;
+ }
+ reuse.length = length;
+ return reuse;
+ }
+
+ static class MutableImpactList extends AbstractList<Impact> implements RandomAccess {
+ int length = 1;
+ Impact[] impacts = new Impact[] {new Impact(Integer.MAX_VALUE, 1L)};
+
+ @Override
+ public Impact get(int index) {
+ return impacts[index];
+ }
+
+ @Override
+ public int size() {
+ return length;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java
new file mode 100644
index 00000000000..a6cb7c3a811
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.codecs.MultiLevelSkipListReader;
+import org.apache.lucene.store.IndexInput;
+
+/**
+ * Implements the skip list reader for block postings format that stores positions and payloads.
+ *
+ * <p>Although this skipper uses MultiLevelSkipListReader as an interface, its definition of skip
+ * position will be a little different.
+ *
+ * <p>For example, when skipInterval = blockSize = 3, df = 2*skipInterval = 6,
+ *
+ * <pre>
+ * 0 1 2 3 4 5
+ * d d d d d d (posting list)
+ * ^ ^ (skip point in MultiLeveSkipWriter)
+ * ^ (skip point in Lucene99SkipWriter)
+ * </pre>
+ *
+ * <p>In this case, MultiLevelSkipListReader will use the last document as a skip point, while
+ * Lucene99SkipReader should assume no skip point will comes.
+ *
+ * <p>If we use the interface directly in Lucene99SkipReader, it may silly try to read another skip
+ * data after the only skip point is loaded.
+ *
+ * <p>To illustrate this, we can call skipTo(d[5]), since skip point d[3] has smaller docId, and
+ * numSkipped+blockSize== df, the MultiLevelSkipListReader will assume the skip list isn't exhausted
+ * yet, and try to load a non-existed skip point
+ *
+ * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)
+ */
+public class Lucene99SkipReader extends MultiLevelSkipListReader {
+ private long[] docPointer;
+ private long[] posPointer;
+ private long[] payPointer;
+ private int[] posBufferUpto;
+ private int[] payloadByteUpto;
+
+ private long lastPosPointer;
+ private long lastPayPointer;
+ private int lastPayloadByteUpto;
+ private long lastDocPointer;
+ private int lastPosBufferUpto;
+
+ public Lucene99SkipReader(
+ IndexInput skipStream,
+ int maxSkipLevels,
+ boolean hasPos,
+ boolean hasOffsets,
+ boolean hasPayloads) {
+ super(skipStream, maxSkipLevels, ForUtil.BLOCK_SIZE, 8);
+ docPointer = new long[maxSkipLevels];
+ if (hasPos) {
+ posPointer = new long[maxSkipLevels];
+ posBufferUpto = new int[maxSkipLevels];
+ if (hasPayloads) {
+ payloadByteUpto = new int[maxSkipLevels];
+ } else {
+ payloadByteUpto = null;
+ }
+ if (hasOffsets || hasPayloads) {
+ payPointer = new long[maxSkipLevels];
+ } else {
+ payPointer = null;
+ }
+ } else {
+ posPointer = null;
+ }
+ }
+
+ /**
+ * Trim original docFreq to tell skipReader read proper number of skip points.
+ *
+ * <p>Since our definition in Lucene99Skip* is a little different from MultiLevelSkip* This
+ * trimmed docFreq will prevent skipReader from: 1. silly reading a non-existed skip point after
+ * the last block boundary 2. moving into the vInt block
+ */
+ protected int trim(int df) {
+ return df % ForUtil.BLOCK_SIZE == 0 ? df - 1 : df;
+ }
+
+ public void init(
+ long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)
+ throws IOException {
+ super.init(skipPointer, trim(df));
+ lastDocPointer = docBasePointer;
+ lastPosPointer = posBasePointer;
+ lastPayPointer = payBasePointer;
+
+ Arrays.fill(docPointer, docBasePointer);
+ if (posPointer != null) {
+ Arrays.fill(posPointer, posBasePointer);
+ if (payPointer != null) {
+ Arrays.fill(payPointer, payBasePointer);
+ }
+ } else {
+ assert posBasePointer == 0;
+ }
+ }
+
+ /**
+ * Returns the doc pointer of the doc to which the last call of {@link
+ * MultiLevelSkipListReader#skipTo(int)} has skipped.
+ */
+ public long getDocPointer() {
+ return lastDocPointer;
+ }
+
+ public long getPosPointer() {
+ return lastPosPointer;
+ }
+
+ public int getPosBufferUpto() {
+ return lastPosBufferUpto;
+ }
+
+ public long getPayPointer() {
+ return lastPayPointer;
+ }
+
+ public int getPayloadByteUpto() {
+ return lastPayloadByteUpto;
+ }
+
+ public int getNextSkipDoc() {
+ return skipDoc[0];
+ }
+
+ @Override
+ protected void seekChild(int level) throws IOException {
+ super.seekChild(level);
+ docPointer[level] = lastDocPointer;
+ if (posPointer != null) {
+ posPointer[level] = lastPosPointer;
+ posBufferUpto[level] = lastPosBufferUpto;
+ if (payloadByteUpto != null) {
+ payloadByteUpto[level] = lastPayloadByteUpto;
+ }
+ if (payPointer != null) {
+ payPointer[level] = lastPayPointer;
+ }
+ }
+ }
+
+ @Override
+ protected void setLastSkipData(int level) {
+ super.setLastSkipData(level);
+ lastDocPointer = docPointer[level];
+
+ if (posPointer != null) {
+ lastPosPointer = posPointer[level];
+ lastPosBufferUpto = posBufferUpto[level];
+ if (payPointer != null) {
+ lastPayPointer = payPointer[level];
+ }
+ if (payloadByteUpto != null) {
+ lastPayloadByteUpto = payloadByteUpto[level];
+ }
+ }
+ }
+
+ @Override
+ protected int readSkipData(int level, IndexInput skipStream) throws IOException {
+ int delta = skipStream.readVInt();
+ docPointer[level] += skipStream.readVLong();
+
+ if (posPointer != null) {
+ posPointer[level] += skipStream.readVLong();
+ posBufferUpto[level] = skipStream.readVInt();
+
+ if (payloadByteUpto != null) {
+ payloadByteUpto[level] = skipStream.readVInt();
+ }
+
+ if (payPointer != null) {
+ payPointer[level] += skipStream.readVLong();
+ }
+ }
+ readImpacts(level, skipStream);
+ return delta;
+ }
+
+ // The default impl skips impacts
+ protected void readImpacts(int level, IndexInput skipStream) throws IOException {
+ skipStream.skipBytes(skipStream.readVInt());
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java
new file mode 100644
index 00000000000..ec50e5e5c16
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
+import org.apache.lucene.codecs.MultiLevelSkipListWriter;
+import org.apache.lucene.index.Impact;
+import org.apache.lucene.store.ByteBuffersDataOutput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+
+/**
+ * Write skip lists with multiple levels, and support skip within block ints.
+ *
+ * <p>Assume that docFreq = 28, skipInterval = blockSize = 12
+ *
+ * <pre>
+ * | block#0 | | block#1 | |vInts|
+ * d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list)
+ * ^ ^ (level 0 skip point)
+ * </pre>
+ *
+ * <p>Note that skipWriter will ignore first document in block#0, since it is useless as a skip
+ * point. Also, we'll never skip into the vInts block, only record skip data at the start its start
+ * point(if it exist).
+ *
+ * <p>For each skip point, we will record: 1. docID in former position, i.e. for position 12, record
+ * docID[11], etc. 2. its related file points(position, payload), 3. related numbers or
+ * uptos(position, payload). 4. start offset.
+ */
+public final class Lucene99SkipWriter extends MultiLevelSkipListWriter {
+ private int[] lastSkipDoc;
+ private long[] lastSkipDocPointer;
+ private long[] lastSkipPosPointer;
+ private long[] lastSkipPayPointer;
+
+ private final IndexOutput docOut;
+ private final IndexOutput posOut;
+ private final IndexOutput payOut;
+
+ private int curDoc;
+ private long curDocPointer;
+ private long curPosPointer;
+ private long curPayPointer;
+ private int curPosBufferUpto;
+ private int curPayloadByteUpto;
+ private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;
+ private boolean fieldHasPositions;
+ private boolean fieldHasOffsets;
+ private boolean fieldHasPayloads;
+
+ public Lucene99SkipWriter(
+ int maxSkipLevels,
+ int blockSize,
+ int docCount,
+ IndexOutput docOut,
+ IndexOutput posOut,
+ IndexOutput payOut) {
+ super(blockSize, 8, maxSkipLevels, docCount);
+ this.docOut = docOut;
+ this.posOut = posOut;
+ this.payOut = payOut;
+
+ lastSkipDoc = new int[maxSkipLevels];
+ lastSkipDocPointer = new long[maxSkipLevels];
+ if (posOut != null) {
+ lastSkipPosPointer = new long[maxSkipLevels];
+ if (payOut != null) {
+ lastSkipPayPointer = new long[maxSkipLevels];
+ }
+ }
+ curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];
+ for (int i = 0; i < maxSkipLevels; ++i) {
+ curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();
+ }
+ }
+
+ public void setField(
+ boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {
+ this.fieldHasPositions = fieldHasPositions;
+ this.fieldHasOffsets = fieldHasOffsets;
+ this.fieldHasPayloads = fieldHasPayloads;
+ }
+
+ // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the
+ // skipper
+ // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of
+ // junk.
+ // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we
+ // save
+ // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.
+ private boolean initialized;
+ long lastDocFP;
+ long lastPosFP;
+ long lastPayFP;
+
+ @Override
+ public void resetSkip() {
+ lastDocFP = docOut.getFilePointer();
+ if (fieldHasPositions) {
+ lastPosFP = posOut.getFilePointer();
+ if (fieldHasOffsets || fieldHasPayloads) {
+ lastPayFP = payOut.getFilePointer();
+ }
+ }
+ if (initialized) {
+ for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {
+ acc.clear();
+ }
+ }
+ initialized = false;
+ }
+
+ private void initSkip() {
+ if (!initialized) {
+ super.resetSkip();
+ Arrays.fill(lastSkipDoc, 0);
+ Arrays.fill(lastSkipDocPointer, lastDocFP);
+ if (fieldHasPositions) {
+ Arrays.fill(lastSkipPosPointer, lastPosFP);
+ if (fieldHasOffsets || fieldHasPayloads) {
+ Arrays.fill(lastSkipPayPointer, lastPayFP);
+ }
+ }
+ // sets of competitive freq,norm pairs should be empty at this point
+ assert Arrays.stream(curCompetitiveFreqNorms)
+ .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)
+ .mapToInt(Collection::size)
+ .sum()
+ == 0;
+ initialized = true;
+ }
+ }
+
+ /** Sets the values for the current skip data. */
+ public void bufferSkip(
+ int doc,
+ CompetitiveImpactAccumulator competitiveFreqNorms,
+ int numDocs,
+ long posFP,
+ long payFP,
+ int posBufferUpto,
+ int payloadByteUpto)
+ throws IOException {
+ initSkip();
+ this.curDoc = doc;
+ this.curDocPointer = docOut.getFilePointer();
+ this.curPosPointer = posFP;
+ this.curPayPointer = payFP;
+ this.curPosBufferUpto = posBufferUpto;
+ this.curPayloadByteUpto = payloadByteUpto;
+ this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms);
+ bufferSkip(numDocs);
+ }
+
+ private final ByteBuffersDataOutput freqNormOut = ByteBuffersDataOutput.newResettableInstance();
+
+ @Override
+ protected void writeSkipData(int level, DataOutput skipBuffer) throws IOException {
+
+ int delta = curDoc - lastSkipDoc[level];
+
+ skipBuffer.writeVInt(delta);
+ lastSkipDoc[level] = curDoc;
+
+ skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);
+ lastSkipDocPointer[level] = curDocPointer;
+
+ if (fieldHasPositions) {
+
+ skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);
+ lastSkipPosPointer[level] = curPosPointer;
+ skipBuffer.writeVInt(curPosBufferUpto);
+
+ if (fieldHasPayloads) {
+ skipBuffer.writeVInt(curPayloadByteUpto);
+ }
+
+ if (fieldHasOffsets || fieldHasPayloads) {
+ skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);
+ lastSkipPayPointer[level] = curPayPointer;
+ }
+ }
+
+ CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];
+ assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;
+ if (level + 1 < numberOfSkipLevels) {
+ curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);
+ }
+ writeImpacts(competitiveFreqNorms, freqNormOut);
+ skipBuffer.writeVInt(Math.toIntExact(freqNormOut.size()));
+ freqNormOut.copyTo(skipBuffer);
+ freqNormOut.reset();
+ competitiveFreqNorms.clear();
+ }
+
+ public static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out)
+ throws IOException {
+ Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();
+ Impact previous = new Impact(0, 0);
+ for (Impact impact : impacts) {
+ assert impact.freq > previous.freq;
+ assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
+ int freqDelta = impact.freq - previous.freq - 1;
+ long normDelta = impact.norm - previous.norm - 1;
+ if (normDelta == 0) {
+ // most of time, norm only increases by 1, so we can fold everything in a single byte
+ out.writeVInt(freqDelta << 1);
+ } else {
+ out.writeVInt((freqDelta << 1) | 1);
+ out.writeZLong(normDelta);
+ }
+ previous = impact;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java
new file mode 100644
index 00000000000..93ebb3d0f9d
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.LongHeap;
+import org.apache.lucene.util.packed.PackedInts;
+
+/** Utility class to encode sequences of 128 small positive integers. */
+final class PForUtil {
+
+ private static final int MAX_EXCEPTIONS = 7;
+
+ // IDENTITY_PLUS_ONE[i] == i + 1
+ private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];
+
+ static {
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ IDENTITY_PLUS_ONE[i] = i + 1;
+ }
+ }
+
+ static boolean allEqual(long[] l) {
+ for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {
+ if (l[i] != l[0]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private final ForUtil forUtil;
+
+ PForUtil(ForUtil forUtil) {
+ assert ForUtil.BLOCK_SIZE <= 256 : "blocksize must fit in one byte. got " + ForUtil.BLOCK_SIZE;
+ this.forUtil = forUtil;
+ }
+
+ /** Encode 128 integers from {@code longs} into {@code out}. */
+ void encode(long[] longs, DataOutput out) throws IOException {
+ // Determine the top MAX_EXCEPTIONS + 1 values
+ final LongHeap top = new LongHeap(MAX_EXCEPTIONS + 1);
+ for (int i = 0; i <= MAX_EXCEPTIONS; ++i) {
+ top.push(longs[i]);
+ }
+ long topValue = top.top();
+ for (int i = MAX_EXCEPTIONS + 1; i < ForUtil.BLOCK_SIZE; ++i) {
+ if (longs[i] > topValue) {
+ topValue = top.updateTop(longs[i]);
+ }
+ }
+
+ long max = 0L;
+ for (int i = 1; i <= top.size(); ++i) {
+ max = Math.max(max, top.get(i));
+ }
+
+ final int maxBitsRequired = PackedInts.bitsRequired(max);
+ // We store the patch on a byte, so we can't decrease the number of bits required by more than 8
+ final int patchedBitsRequired =
+ Math.max(PackedInts.bitsRequired(topValue), maxBitsRequired - 8);
+ int numExceptions = 0;
+ final long maxUnpatchedValue = (1L << patchedBitsRequired) - 1;
+ for (int i = 2; i <= top.size(); ++i) {
+ if (top.get(i) > maxUnpatchedValue) {
+ numExceptions++;
+ }
+ }
+ final byte[] exceptions = new byte[numExceptions * 2];
+ if (numExceptions > 0) {
+ int exceptionCount = 0;
+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
+ if (longs[i] > maxUnpatchedValue) {
+ exceptions[exceptionCount * 2] = (byte) i;
+ exceptions[exceptionCount * 2 + 1] = (byte) (longs[i] >>> patchedBitsRequired);
+ longs[i] &= maxUnpatchedValue;
+ exceptionCount++;
+ }
+ }
+ assert exceptionCount == numExceptions : exceptionCount + " " + numExceptions;
+ }
+
+ if (allEqual(longs) && maxBitsRequired <= 8) {
+ for (int i = 0; i < numExceptions; ++i) {
+ exceptions[2 * i + 1] =
+ (byte) (Byte.toUnsignedLong(exceptions[2 * i + 1]) << patchedBitsRequired);
+ }
+ out.writeByte((byte) (numExceptions << 5));
+ out.writeVLong(longs[0]);
+ } else {
+ final int token = (numExceptions << 5) | patchedBitsRequired;
+ out.writeByte((byte) token);
+ forUtil.encode(longs, patchedBitsRequired, out);
+ }
+ out.writeBytes(exceptions, exceptions.length);
+ }
+
+ /** Decode 128 integers into {@code ints}. */
+ void decode(DataInput in, long[] longs) throws IOException {
+ final int token = Byte.toUnsignedInt(in.readByte());
+ final int bitsPerValue = token & 0x1f;
+ final int numExceptions = token >>> 5;
+ if (bitsPerValue == 0) {
+ Arrays.fill(longs, 0, ForUtil.BLOCK_SIZE, in.readVLong());
+ } else {
+ forUtil.decode(bitsPerValue, in, longs);
+ }
+ for (int i = 0; i < numExceptions; ++i) {
+ longs[Byte.toUnsignedInt(in.readByte())] |=
+ Byte.toUnsignedLong(in.readByte()) << bitsPerValue;
+ }
+ }
+
+ /** Skip 128 integers. */
+ void skip(DataInput in) throws IOException {
+ final int token = Byte.toUnsignedInt(in.readByte());
+ final int bitsPerValue = token & 0x1f;
+ final int numExceptions = token >>> 5;
+ if (bitsPerValue == 0) {
+ in.readVLong();
+ in.skipBytes((numExceptions << 1));
+ } else {
+ in.skipBytes(forUtil.numBytes(bitsPerValue) + (numExceptions << 1));
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py
new file mode 100644
index 00000000000..cd81d493570
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py
@@ -0,0 +1,524 @@
+#! /usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from math import gcd
+
+"""Code generation for ForUtil.java"""
+
+MAX_SPECIALIZED_BITS_PER_VALUE = 24
+OUTPUT_FILE = "ForUtil.java"
+PRIMITIVE_SIZE = [8, 16, 32]
+HEADER = """// This file has been automatically generated, DO NOT EDIT
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import java.io.IOException;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+// Inspired from https://fulmicoton.com/posts/bitpacking/
+// Encodes multiple integers in a long to get SIMD-like speedups.
+// If bitsPerValue <= 8 then we pack 8 ints per long
+// else if bitsPerValue <= 16 we pack 4 ints per long
+// else we pack 2 ints per long
+final class ForUtil {
+
+ static final int BLOCK_SIZE = 128;
+ private static final int BLOCK_SIZE_LOG2 = 7;
+
+ private static long expandMask32(long mask32) {
+ return mask32 | (mask32 << 32);
+ }
+
+ private static long expandMask16(long mask16) {
+ return expandMask32(mask16 | (mask16 << 16));
+ }
+
+ private static long expandMask8(long mask8) {
+ return expandMask16(mask8 | (mask8 << 8));
+ }
+
+ private static long mask32(int bitsPerValue) {
+ return expandMask32((1L << bitsPerValue) - 1);
+ }
+
+ private static long mask16(int bitsPerValue) {
+ return expandMask16((1L << bitsPerValue) - 1);
+ }
+
+ private static long mask8(int bitsPerValue) {
+ return expandMask8((1L << bitsPerValue) - 1);
+ }
+
+ private static void expand8(long[] arr) {
+ for (int i = 0; i < 16; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 56) & 0xFFL;
+ arr[16 + i] = (l >>> 48) & 0xFFL;
+ arr[32 + i] = (l >>> 40) & 0xFFL;
+ arr[48 + i] = (l >>> 32) & 0xFFL;
+ arr[64 + i] = (l >>> 24) & 0xFFL;
+ arr[80 + i] = (l >>> 16) & 0xFFL;
+ arr[96 + i] = (l >>> 8) & 0xFFL;
+ arr[112 + i] = l & 0xFFL;
+ }
+ }
+
+ private static void expand8To32(long[] arr) {
+ for (int i = 0; i < 16; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 24) & 0x000000FF000000FFL;
+ arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;
+ arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;
+ arr[48 + i] = l & 0x000000FF000000FFL;
+ }
+ }
+
+ private static void collapse8(long[] arr) {
+ for (int i = 0; i < 16; ++i) {
+ arr[i] =
+ (arr[i] << 56)
+ | (arr[16 + i] << 48)
+ | (arr[32 + i] << 40)
+ | (arr[48 + i] << 32)
+ | (arr[64 + i] << 24)
+ | (arr[80 + i] << 16)
+ | (arr[96 + i] << 8)
+ | arr[112 + i];
+ }
+ }
+
+ private static void expand16(long[] arr) {
+ for (int i = 0; i < 32; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 48) & 0xFFFFL;
+ arr[32 + i] = (l >>> 32) & 0xFFFFL;
+ arr[64 + i] = (l >>> 16) & 0xFFFFL;
+ arr[96 + i] = l & 0xFFFFL;
+ }
+ }
+
+ private static void expand16To32(long[] arr) {
+ for (int i = 0; i < 32; ++i) {
+ long l = arr[i];
+ arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;
+ arr[32 + i] = l & 0x0000FFFF0000FFFFL;
+ }
+ }
+
+ private static void collapse16(long[] arr) {
+ for (int i = 0; i < 32; ++i) {
+ arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];
+ }
+ }
+
+ private static void expand32(long[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ long l = arr[i];
+ arr[i] = l >>> 32;
+ arr[64 + i] = l & 0xFFFFFFFFL;
+ }
+ }
+
+ private static void collapse32(long[] arr) {
+ for (int i = 0; i < 64; ++i) {
+ arr[i] = (arr[i] << 32) | arr[64 + i];
+ }
+ }
+
+ private static void prefixSum8(long[] arr, long base) {
+ expand8To32(arr);
+ prefixSum32(arr, base);
+ }
+
+ private static void prefixSum16(long[] arr, long base) {
+ // We need to move to the next primitive size to avoid overflows
+ expand16To32(arr);
+ prefixSum32(arr, base);
+ }
+
+ private static void prefixSum32(long[] arr, long base) {
+ arr[0] += base << 32;
+ innerPrefixSum32(arr);
+ expand32(arr);
+ final long l = arr[BLOCK_SIZE/2-1];
+ for (int i = BLOCK_SIZE/2; i < BLOCK_SIZE; ++i) {
+ arr[i] += l;
+ }
+ }
+
+ // For some reason unrolling seems to help
+ private static void innerPrefixSum32(long[] arr) {
+ arr[1] += arr[0];
+ arr[2] += arr[1];
+ arr[3] += arr[2];
+ arr[4] += arr[3];
+ arr[5] += arr[4];
+ arr[6] += arr[5];
+ arr[7] += arr[6];
+ arr[8] += arr[7];
+ arr[9] += arr[8];
+ arr[10] += arr[9];
+ arr[11] += arr[10];
+ arr[12] += arr[11];
+ arr[13] += arr[12];
+ arr[14] += arr[13];
+ arr[15] += arr[14];
+ arr[16] += arr[15];
+ arr[17] += arr[16];
+ arr[18] += arr[17];
+ arr[19] += arr[18];
+ arr[20] += arr[19];
+ arr[21] += arr[20];
+ arr[22] += arr[21];
+ arr[23] += arr[22];
+ arr[24] += arr[23];
+ arr[25] += arr[24];
+ arr[26] += arr[25];
+ arr[27] += arr[26];
+ arr[28] += arr[27];
+ arr[29] += arr[28];
+ arr[30] += arr[29];
+ arr[31] += arr[30];
+ arr[32] += arr[31];
+ arr[33] += arr[32];
+ arr[34] += arr[33];
+ arr[35] += arr[34];
+ arr[36] += arr[35];
+ arr[37] += arr[36];
+ arr[38] += arr[37];
+ arr[39] += arr[38];
+ arr[40] += arr[39];
+ arr[41] += arr[40];
+ arr[42] += arr[41];
+ arr[43] += arr[42];
+ arr[44] += arr[43];
+ arr[45] += arr[44];
+ arr[46] += arr[45];
+ arr[47] += arr[46];
+ arr[48] += arr[47];
+ arr[49] += arr[48];
+ arr[50] += arr[49];
+ arr[51] += arr[50];
+ arr[52] += arr[51];
+ arr[53] += arr[52];
+ arr[54] += arr[53];
+ arr[55] += arr[54];
+ arr[56] += arr[55];
+ arr[57] += arr[56];
+ arr[58] += arr[57];
+ arr[59] += arr[58];
+ arr[60] += arr[59];
+ arr[61] += arr[60];
+ arr[62] += arr[61];
+ arr[63] += arr[62];
+ }
+
+ private final long[] tmp = new long[BLOCK_SIZE / 2];
+
+ /** Encode 128 integers from {@code longs} into {@code out}. */
+ void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
+ final int nextPrimitive;
+ final int numLongs;
+ if (bitsPerValue <= 8) {
+ nextPrimitive = 8;
+ numLongs = BLOCK_SIZE / 8;
+ collapse8(longs);
+ } else if (bitsPerValue <= 16) {
+ nextPrimitive = 16;
+ numLongs = BLOCK_SIZE / 4;
+ collapse16(longs);
+ } else {
+ nextPrimitive = 32;
+ numLongs = BLOCK_SIZE / 2;
+ collapse32(longs);
+ }
+
+ final int numLongsPerShift = bitsPerValue * 2;
+ int idx = 0;
+ int shift = nextPrimitive - bitsPerValue;
+ for (int i = 0; i < numLongsPerShift; ++i) {
+ tmp[i] = longs[idx++] << shift;
+ }
+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
+ for (int i = 0; i < numLongsPerShift; ++i) {
+ tmp[i] |= longs[idx++] << shift;
+ }
+ }
+
+ final int remainingBitsPerLong = shift + bitsPerValue;
+ final long maskRemainingBitsPerLong;
+ if (nextPrimitive == 8) {
+ maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];
+ } else if (nextPrimitive == 16) {
+ maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];
+ } else {
+ maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];
+ }
+
+ int tmpIdx = 0;
+ int remainingBitsPerValue = bitsPerValue;
+ while (idx < numLongs) {
+ if (remainingBitsPerValue >= remainingBitsPerLong) {
+ remainingBitsPerValue -= remainingBitsPerLong;
+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;
+ if (remainingBitsPerValue == 0) {
+ idx++;
+ remainingBitsPerValue = bitsPerValue;
+ }
+ } else {
+ final long mask1, mask2;
+ if (nextPrimitive == 8) {
+ mask1 = MASKS8[remainingBitsPerValue];
+ mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];
+ } else if (nextPrimitive == 16) {
+ mask1 = MASKS16[remainingBitsPerValue];
+ mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];
+ } else {
+ mask1 = MASKS32[remainingBitsPerValue];
+ mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];
+ }
+ tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);
+ remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;
+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;
+ }
+ }
+
+ for (int i = 0; i < numLongsPerShift; ++i) {
+ out.writeLong(tmp[i]);
+ }
+ }
+
+ /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */
+ int numBytes(int bitsPerValue) {
+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
+ }
+
+ private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)
+ throws IOException {
+ final int numLongs = bitsPerValue << 1;
+ in.readLongs(tmp, 0, numLongs);
+ final long mask = MASKS32[bitsPerValue];
+ int longsIdx = 0;
+ int shift = 32 - bitsPerValue;
+ for (; shift >= 0; shift -= bitsPerValue) {
+ shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);
+ longsIdx += numLongs;
+ }
+ final int remainingBitsPerLong = shift + bitsPerValue;
+ final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];
+ int tmpIdx = 0;
+ int remainingBits = remainingBitsPerLong;
+ for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {
+ int b = bitsPerValue - remainingBits;
+ long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
+ while (b >= remainingBitsPerLong) {
+ b -= remainingBitsPerLong;
+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;
+ }
+ if (b > 0) {
+ l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];
+ remainingBits = remainingBitsPerLong - b;
+ } else {
+ remainingBits = remainingBitsPerLong;
+ }
+ longs[longsIdx] = l;
+ }
+ }
+
+ /**
+ * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which
+ * generates SIMD instructions for it in order to shift multiple longs at once.
+ */
+ private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {
+ for (int i = 0; i < count; ++i) {
+ b[bi + i] = (a[i] >>> shift) & mask;
+ }
+ }
+
+"""
+
+def writeRemainderWithSIMDOptimize(bpv, next_primitive, remaining_bits_per_long, o, num_values, f):
+ iteration = 1
+ num_longs = bpv * num_values / remaining_bits_per_long
+ while num_longs % 2 == 0 and num_values % 2 == 0:
+ num_longs /= 2
+ num_values /= 2
+ iteration *= 2
+
+ f.write(' shiftLongs(tmp, %d, tmp, 0, 0, MASK%d_%d);\n' % (iteration * num_longs, next_primitive, remaining_bits_per_long))
+ f.write(' for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n' %(o, iteration, num_longs, num_values))
+ tmp_idx = 0
+ b = bpv
+ b -= remaining_bits_per_long
+ f.write(' long l0 = tmp[tmpIdx + %d] << %d;\n' %(tmp_idx, b))
+ tmp_idx += 1
+ while b >= remaining_bits_per_long:
+ b -= remaining_bits_per_long
+ f.write(' l0 |= tmp[tmpIdx + %d] << %d;\n' %(tmp_idx, b))
+ tmp_idx += 1
+ f.write(' longs[longsIdx + 0] = l0;\n')
+ f.write(' }\n')
+
+
+def writeRemainder(bpv, next_primitive, remaining_bits_per_long, o, num_values, f):
+ iteration = 1
+ num_longs = bpv * num_values / remaining_bits_per_long
+ while num_longs % 2 == 0 and num_values % 2 == 0:
+ num_longs /= 2
+ num_values /= 2
+ iteration *= 2
+ f.write(' for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n' %(o, iteration, num_longs, num_values))
+ i = 0
+ remaining_bits = 0
+ tmp_idx = 0
+ for i in range(int(num_values)):
+ b = bpv
+ if remaining_bits == 0:
+ b -= remaining_bits_per_long
+ f.write(' long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b))
+ else:
+ b -= remaining_bits
+ f.write(' long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits, b))
+ tmp_idx += 1
+ while b >= remaining_bits_per_long:
+ b -= remaining_bits_per_long
+ f.write(' l%d |= (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b))
+ tmp_idx += 1
+ if b > 0:
+ f.write(' l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n' %(i, tmp_idx, remaining_bits_per_long-b, next_primitive, b))
+ remaining_bits = remaining_bits_per_long-b
+ f.write(' longs[longsIdx + %d] = l%d;\n' %(i, i))
+ f.write(' }\n')
+
+
+def writeDecode(bpv, f):
+ next_primitive = 32
+ if bpv <= 8:
+ next_primitive = 8
+ elif bpv <= 16:
+ next_primitive = 16
+ f.write(' private static void decode%d(DataInput in, long[] tmp, long[] longs) throws IOException {\n' %bpv)
+ num_values_per_long = 64 / next_primitive
+ if bpv == next_primitive:
+ f.write(' in.readLongs(longs, 0, %d);\n' %(bpv*2))
+ else:
+ f.write(' in.readLongs(tmp, 0, %d);\n' %(bpv*2))
+ shift = next_primitive - bpv
+ o = 0
+ while shift >= 0:
+ f.write(' shiftLongs(tmp, %d, longs, %d, %d, MASK%d_%d);\n' %(bpv*2, o, shift, next_primitive, bpv))
+ o += bpv*2
+ shift -= bpv
+ if shift + bpv > 0:
+ if bpv % (next_primitive % bpv) == 0:
+ writeRemainderWithSIMDOptimize(bpv, next_primitive, shift + bpv, o, 128/num_values_per_long - o, f)
+ else:
+ writeRemainder(bpv, next_primitive, shift + bpv, o, 128/num_values_per_long - o, f)
+ f.write(' }\n')
+
+
+if __name__ == '__main__':
+ f = open(OUTPUT_FILE, 'w')
+ f.write(HEADER)
+ for primitive_size in PRIMITIVE_SIZE:
+ f.write(' private static final long[] MASKS%d = new long[%d];\n' %(primitive_size, primitive_size))
+ f.write('\n')
+ f.write(' static {\n')
+ for primitive_size in PRIMITIVE_SIZE:
+ f.write(' for (int i = 0; i < %d; ++i) {\n' %primitive_size)
+ f.write(' MASKS%d[i] = mask%d(i);\n' %(primitive_size, primitive_size))
+ f.write(' }\n')
+ f.write(' }')
+ f.write("""
+ // mark values in array as final longs to avoid the cost of reading array, arrays should only be
+ // used when the idx is a variable
+""")
+ for primitive_size in PRIMITIVE_SIZE:
+ for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)):
+ if bpv * 2 != primitive_size or primitive_size == 8:
+ f.write(' private static final long MASK%d_%d = MASKS%d[%d];\n' %(primitive_size, bpv, primitive_size, bpv))
+
+ f.write("""
+ /** Decode 128 integers into {@code longs}. */
+ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
+ switch (bitsPerValue) {
+""")
+ for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):
+ next_primitive = 32
+ if bpv <= 8:
+ next_primitive = 8
+ elif bpv <= 16:
+ next_primitive = 16
+ f.write(' case %d:\n' %bpv)
+ f.write(' decode%d(in, tmp, longs);\n' %bpv)
+ f.write(' expand%d(longs);\n' %next_primitive)
+ f.write(' break;\n')
+ f.write(' default:\n')
+ f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n')
+ f.write(' expand32(longs);\n')
+ f.write(' break;\n')
+ f.write(' }\n')
+ f.write(' }\n')
+
+ f.write("""
+ /**
+ * Delta-decode 128 integers into {@code longs}.
+ */
+ void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs) throws IOException {
+ switch (bitsPerValue) {
+""")
+ for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):
+ next_primitive = 32
+ if bpv <= 8:
+ next_primitive = 8
+ elif bpv <= 16:
+ next_primitive = 16
+ f.write(' case %d:\n' %bpv)
+ f.write(' decode%d(in, tmp, longs);\n' %bpv)
+ f.write(' prefixSum%d(longs, base);\n' %next_primitive)
+ f.write(' break;\n')
+ f.write(' default:\n')
+ f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n')
+ f.write(' prefixSum32(longs, base);\n')
+ f.write(' break;\n')
+ f.write(' }\n')
+ f.write(' }\n')
+
+ f.write('\n')
+ for i in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):
+ writeDecode(i, f)
+ if i < MAX_SPECIALIZED_BITS_PER_VALUE:
+ f.write('\n')
+
+ f.write('}\n')
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
index 2511e3be0da..2df2cfc9a30 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
@@ -151,15 +151,15 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A
+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term dictionary}. A
* dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Frequency data}. For
* each term in the dictionary, the numbers of all the documents that contain that term, and
* the frequency of the term in that document, unless frequencies are omitted ({@link
* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Proximity data}. For
* each term in the dictionary, the positions that the term occurs in each document. Note that
* this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index 853e8f9b0c7..ea9df193413 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -21,8 +21,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -401,10 +401,10 @@ public class PhraseQuery extends Query {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene90PostingsReader.BlockImpactsPostingsEnum#nextPosition()}.
+ * Lucene99PostingsReader.BlockImpactsPostingsEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene90PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene99PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
@@ -413,7 +413,7 @@ public class PhraseQuery extends Query {
/**
* Number of simple operations in {@link
- * Lucene90PostingsReader.BlockImpactsPostingsEnum#nextPosition()} when no seek or buffer refill
+ * Lucene99PostingsReader.BlockImpactsPostingsEnum#nextPosition()} when no seek or buffer refill
* is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 15681e8a2e5..0fd87daff7d 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat
+org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java
new file mode 100644
index 00000000000..29448ff1e41
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+public class TestForDeltaUtil extends LuceneTestCase {
+
+ public void testEncodeDecode() throws IOException {
+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
+
+ for (int i = 0; i < iterations; ++i) {
+ final int bpv = TestUtil.nextInt(random(), 1, 31 - 7);
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ values[i * ForUtil.BLOCK_SIZE + j] =
+ RandomNumbers.randomIntBetween(random(), 1, (int) PackedInts.maxValue(bpv));
+ }
+ }
+
+ final Directory d = new ByteBuffersDirectory();
+ final long endPointer;
+
+ {
+ // encode
+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());
+
+ for (int i = 0; i < iterations; ++i) {
+ long[] source = new long[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ }
+ forDeltaUtil.encodeDeltas(source, out);
+ }
+ endPointer = out.getFilePointer();
+ out.close();
+ }
+
+ {
+ // decode
+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());
+ for (int i = 0; i < iterations; ++i) {
+ long base = 0;
+ final long[] restored = new long[ForUtil.BLOCK_SIZE];
+ forDeltaUtil.decodeAndPrefixSum(in, base, restored);
+ final long[] expected = new long[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ expected[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ if (j > 0) {
+ expected[j] += expected[j - 1];
+ } else {
+ expected[j] += base;
+ }
+ }
+ assertArrayEquals(Arrays.toString(restored), expected, restored);
+ }
+ assertEquals(endPointer, in.getFilePointer());
+ in.close();
+ }
+
+ d.close();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestForUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java
index 363226423ee..2f179c743af 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestForUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.codecs.lucene99;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java
similarity index 91%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PostingsFormat.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java
index 90a9ebe5369..99c0e0a6ae2 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java
@@ -14,7 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.codecs.lucene99;
+
+import static org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.readImpacts;
import java.io.IOException;
import java.util.Arrays;
@@ -22,9 +24,9 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.lucene90.Lucene90ScoreSkipReader.MutableImpactList;
import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
import org.apache.lucene.codecs.lucene90.blocktree.Stats;
+import org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -40,8 +42,8 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
-public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene90PostingsFormat());
+public class TestLucene99PostingsFormat extends BasePostingsFormatTestCase {
+ private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene99PostingsFormat());
@Override
protected Codec getCodec() {
@@ -77,7 +79,7 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
expectThrows(
IllegalArgumentException.class,
() -> {
- new Lucene90PostingsFormat(minItemsInBlock, maxItemsInBlock);
+ new Lucene99PostingsFormat(minItemsInBlock, maxItemsInBlock);
});
}
@@ -131,13 +133,12 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
}
try (Directory dir = newDirectory()) {
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
- Lucene90SkipWriter.writeImpacts(acc, out);
+ Lucene99SkipWriter.writeImpacts(acc, out);
}
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
byte[] b = new byte[Math.toIntExact(in.length())];
in.readBytes(b, 0, b.length);
- List<Impact> impacts2 =
- Lucene90ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
+ List<Impact> impacts2 = readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
assertEquals(impacts, impacts2);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java
similarity index 73%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestPForUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java
index a7b37030ad1..16f719e93b2 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestPForUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90;
+package org.apache.lucene.codecs.lucene99;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
@@ -62,42 +62,6 @@ public class TestPForUtil extends LuceneTestCase {
d.close();
}
- public void testDeltaEncodeDecode() throws IOException {
- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
- // cap at 31 - 7 bpv to ensure we don't overflow when working with deltas (i.e., 128 24 bit
- // values treated as deltas will result in a final value that can fit in 31 bits)
- final int[] values = createTestData(iterations, 31 - 7);
-
- final Directory d = new ByteBuffersDirectory();
- final long endPointer = encodeTestData(iterations, values, d);
-
- IndexInput in = d.openInput("test.bin", IOContext.READONCE);
- final PForUtil pForUtil = new PForUtil(new ForUtil());
- for (int i = 0; i < iterations; ++i) {
- if (random().nextInt(5) == 0) {
- pForUtil.skip(in);
- continue;
- }
- long base = 0;
- final long[] restored = new long[ForUtil.BLOCK_SIZE];
- pForUtil.decodeAndPrefixSum(in, base, restored);
- final long[] expected = new long[ForUtil.BLOCK_SIZE];
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- expected[j] = values[i * ForUtil.BLOCK_SIZE + j];
- if (j > 0) {
- expected[j] += expected[j - 1];
- } else {
- expected[j] += base;
- }
- }
- assertArrayEquals(Arrays.toString(restored), expected, restored);
- }
- assertEquals(endPointer, in.getFilePointer());
- in.close();
-
- d.close();
- }
-
private int[] createTestData(int iterations, int maxBpv) {
final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
index 414d360d5d1..2a5d8fffd01 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@@ -261,10 +261,10 @@ class TermIntervalsSource extends IntervalsSource {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene90PostingsReader.EverythingEnum#nextPosition()}.
+ * Lucene99PostingsReader.EverythingEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene90PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene99PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
@@ -272,7 +272,7 @@ class TermIntervalsSource extends IntervalsSource {
private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
/**
- * Number of simple operations in {@link Lucene90PostingsReader.EverythingEnum#nextPosition()}
+ * Number of simple operations in {@link Lucene99PostingsReader.EverythingEnum#nextPosition()}
* when no seek or buffer refill is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/suggest/src/java/module-info.java b/lucene/suggest/src/java/module-info.java
index 4444a9dd2f1..673fa391213 100644
--- a/lucene/suggest/src/java/module-info.java
+++ b/lucene/suggest/src/java/module-info.java
@@ -30,7 +30,8 @@ module org.apache.lucene.suggest {
provides org.apache.lucene.codecs.PostingsFormat with
org.apache.lucene.search.suggest.document.Completion50PostingsFormat,
org.apache.lucene.search.suggest.document.Completion84PostingsFormat,
- org.apache.lucene.search.suggest.document.Completion90PostingsFormat;
+ org.apache.lucene.search.suggest.document.Completion90PostingsFormat,
+ org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
provides org.apache.lucene.analysis.TokenFilterFactory with
org.apache.lucene.search.suggest.analyzing.SuggestStopFilterFactory;
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java
index 02536cc7f26..e53e991db22 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java
@@ -22,7 +22,7 @@ import org.apache.lucene.codecs.PostingsFormat;
* {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code
* org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat}. This format is only used for
* backward-compatibility of the index format and cannot be used to write data, use {@link
- * Completion90PostingsFormat} on new indices.
+ * Completion99PostingsFormat} on new indices.
*
* @lucene.experimental
*/
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java
index 8551fc3a80a..76e29811834 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java
@@ -22,7 +22,7 @@ import org.apache.lucene.codecs.PostingsFormat;
* {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code
* org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat}. This format is only used for
* backward-compatibility of the index format and cannot be used to write data, use {@link
- * Completion90PostingsFormat} on new indices.
+ * Completion99PostingsFormat} on new indices.
*
* @lucene.experimental
*/
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java
index cf2a9103aa7..db45303bcd7 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java
@@ -19,8 +19,10 @@ package org.apache.lucene.search.suggest.document;
import org.apache.lucene.codecs.PostingsFormat;
/**
- * {@link CompletionPostingsFormat} for {@link
- * org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat}
+ * {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code
+ * org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat}. This format is only used for
+ * backward-compatibility of the index format and cannot be used to write data, use {@link
+ * Completion90PostingsFormat} on new indices.
*
* @lucene.experimental
*/
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
new file mode 100644
index 00000000000..4449d754e29
--- /dev/null
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search.suggest.document;
+
+import org.apache.lucene.codecs.PostingsFormat;
+
+/**
+ * {@link CompletionPostingsFormat} for {@link
+ * org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}
+ *
+ * @lucene.experimental
+ */
+public class Completion99PostingsFormat extends CompletionPostingsFormat {
+ /** Creates a {@link Completion99PostingsFormat} that will load the completion FST on-heap. */
+ public Completion99PostingsFormat() {
+ this(FSTLoadMode.ON_HEAP);
+ }
+
+ /**
+ * Creates a {@link Completion99PostingsFormat} that will use the provided <code>fstLoadMode
+ * </code> to determine if the completion FST should be loaded on or off heap.
+ */
+ public Completion99PostingsFormat(FSTLoadMode fstLoadMode) {
+ super("Completion99", fstLoadMode);
+ }
+
+ @Override
+ protected PostingsFormat delegatePostingsFormat() {
+ return PostingsFormat.forName("Lucene99");
+ }
+}
diff --git a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index a4070471047..81d1ed9465d 100644
--- a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -33,3 +33,4 @@
org.apache.lucene.search.suggest.document.Completion50PostingsFormat
org.apache.lucene.search.suggest.document.Completion84PostingsFormat
org.apache.lucene.search.suggest.document.Completion90PostingsFormat
+org.apache.lucene.search.suggest.document.Completion99PostingsFormat
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
index 682363f29cb..30201a8eac8 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
@@ -965,7 +965,7 @@ public class TestSuggestField extends LuceneTestCase {
new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {
CompletionPostingsFormat.FSTLoadMode fstLoadMode =
RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());
- PostingsFormat postingsFormat = new Completion90PostingsFormat(fstLoadMode);
+ PostingsFormat postingsFormat = new Completion99PostingsFormat(fstLoadMode);
@Override
public PostingsFormat postingsFormat() {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
index 3c21de0f128..0a0c8f673db 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
@@ -28,9 +28,9 @@ import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;
import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; // javadocs
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -38,7 +38,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene90PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
+ * Customized version of {@link Lucene99PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
*/
public final class LuceneFixedGap extends PostingsFormat {
final int termIndexInterval;
@@ -54,7 +54,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene90PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene99PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -91,7 +91,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene90PostingsReader(state);
+ PostingsReaderBase postings = new Lucene99PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
index 6a38882c153..a864263b7c3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; // javadocs
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene90PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene99PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval, but forcing high docfreq terms to be indexed
* terms.
*/
@@ -59,7 +59,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene90PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene99PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -100,7 +100,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene90PostingsReader(state);
+ PostingsReaderBase postings = new Lucene99PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
index 838d2968c7f..667acf0ceb3 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; // javadocs
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene90PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene99PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval.
*/
public final class LuceneVarGapFixedInterval extends PostingsFormat {
@@ -56,7 +56,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene90PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene99PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -95,7 +95,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene90PostingsReader(state);
+ PostingsReaderBase postings = new Lucene99PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
index 9243ce95059..10e5ec96332 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
@@ -35,10 +35,10 @@ import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.codecs.memory.FSTTermsReader;
import org.apache.lucene.codecs.memory.FSTTermsWriter;
import org.apache.lucene.index.FieldInfo;
@@ -121,7 +121,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
random.nextInt(); // consume a random for buffersize
- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
final FieldsConsumer fields;
final int t1 = random.nextInt(4);
@@ -289,7 +289,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);
}
- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
final FieldsProducer fields;
final int t1 = random.nextInt(4);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
index 7291c7e54da..0c469b392a2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
import org.apache.lucene.codecs.uniformsplit.BlockDecoder;
import org.apache.lucene.codecs.uniformsplit.BlockEncoder;
import org.apache.lucene.codecs.uniformsplit.IndexDictionary;
@@ -67,7 +67,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState segmentWriteState) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(segmentWriteState);
+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(segmentWriteState);
boolean success = false;
try {
FieldsConsumer fieldsConsumer = createFieldsConsumer(segmentWriteState, postingsWriter);
@@ -145,7 +145,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState segmentReadState) throws IOException {
- PostingsReaderBase postingsReader = new Lucene90PostingsReader(segmentReadState);
+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(segmentReadState);
boolean success = false;
try {
FieldsProducer fieldsProducer = createFieldsProducer(segmentReadState, postingsReader);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
index b1c5e8d9b93..6d1d8573e5e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
@@ -54,9 +54,9 @@ import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -1244,7 +1244,7 @@ public final class TestUtil {
* Lucene.
*/
public static PostingsFormat getDefaultPostingsFormat() {
- return new Lucene90PostingsFormat();
+ return new Lucene99PostingsFormat();
}
/**
@@ -1255,7 +1255,7 @@ public final class TestUtil {
*/
public static PostingsFormat getDefaultPostingsFormat(
int minItemsPerBlock, int maxItemsPerBlock) {
- return new Lucene90PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
+ return new Lucene99PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
}
/** Returns a random postings format that supports term ordinals */