Apache Lucene open-source search software
Remove patching for doc blocks. (#12741)
Jakub Slowinski8ae598bae593
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradleindex 9ada40c8dde..96f5bfdf974 100644--- a/gradle/generation/forUtil.gradle+++ b/gradle/generation/forUtil.gradle@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {description "Regenerate gen_ForUtil.py"group "generation"- def genDir = file("src/java/org/apache/lucene/codecs/lucene90")+ def genDir = file("src/java/org/apache/lucene/codecs/lucene99")def genScript = file("${genDir}/gen_ForUtil.py")def genOutput = file("${genDir}/ForUtil.java")@@ -44,7 +44,7 @@ configure(project(":lucene:core")) {configure(project(":lucene:backward-codecs")) {- task generateForUtilInternal() {+ task generateForUtil84Internal() {description "Regenerate gen_ForUtil.py"group "generation"@@ -64,6 +64,28 @@ configure(project(":lucene:backward-codecs")) {}}- regenerate.dependsOn wrapWithPersistentChecksums(generateForUtilInternal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil84Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])++ task generateForUtil90Internal() {+ description "Regenerate gen_ForUtil.py"+ group "generation"++ def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene90")+ def genScript = file("${genDir}/gen_ForUtil.py")+ def genOutput = file("${genDir}/ForUtil.java")++ inputs.file genScript+ outputs.file genOutput++ doLast {+ quietExec {+ workingDir genDir+ executable project.externalTool("python3")+ args = [ '-B', genScript ]+ }+ }+ }++ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil90Internal, [ andThenTasks: ["spotlessJava", "spotlessJavaApply"] ])}diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 0ee491bbec5..12b058fe14e 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -256,11 +256,12 @@ Optimizations* GITHUB#12719: Top-level conjunctions that are not sorted by score now have aspecialized bulk scorer. (Adrien Grand)+* GITHUB#12696: Change Postings back to using FOR in Lucene99PostingsFormat. Freqs, positions and offset keep using PFOR. (Jakub Slowinski)+* GITHUB#1052: Faster merging of terms enums. (Adrien Grand)* GITHUB#11903: Faster sort on high-cardinality string fields. (Adrien Grand)-Changes in runtime behavior---------------------diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil84.jsonsimilarity index 100%rename from lucene/backward-codecs/src/generated/checksums/generateForUtil.jsonrename to lucene/backward-codecs/src/generated/checksums/generateForUtil84.jsondiff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil90.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil90.jsonnew file mode 100644index 00000000000..15fd411b1cb--- /dev/null+++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil90.json@@ -0,0 +1,4 @@+{+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.java": "861cab516c7424e6323831c16f0f521499391a90",+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.py": "b66e2f8012759d6d5ce0d73fabb329ae4a391aa0"+}\ No newline at end of filediff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.javaindex 689e0a7d5b4..992ad22a773 100644--- a/lucene/backward-codecs/src/java/module-info.java+++ b/lucene/backward-codecs/src/java/module-info.java@@ -42,7 +42,8 @@ module org.apache.lucene.backward_codecs {org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;provides org.apache.lucene.codecs.PostingsFormat withorg.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,- org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;+ org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,+ org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;provides org.apache.lucene.codecs.KnnVectorsFormat withorg.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.javasimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.javaindex a58c84e314b..c889c8b8626 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/ForUtil.java@@ -16,7 +16,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import org.apache.lucene.store.DataInput;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.javaindex b9465985063..77280dcff63 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90Codec.java@@ -35,7 +35,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.javasimilarity index 94%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsFormat.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.javaindex de0d9c381a8..0fceeb47296 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import org.apache.lucene.codecs.BlockTermState;@@ -24,7 +24,6 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.MultiLevelSkipListWriter;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.codecs.PostingsWriterBase;import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;import org.apache.lucene.index.IndexOptions;@@ -36,7 +35,9 @@ import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.packed.PackedInts;/**- * Lucene 5.0 postings format, which encodes postings in packed integer blocks for fast decode.+ * Lucene 9.0 postings format, which encodes postings in packed integer blocks for fast decode.+ *+ * <p>Note: Lucene90PostingsFormat is now READ ONLY.** <p>Basic idea:*@@ -371,30 +372,11 @@ public final class Lucene90PostingsFormat extends PostingsFormat {// Increment version to change itstatic final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-- private final int minTermBlockSize;- private final int maxTermBlockSize;+ static final int VERSION_CURRENT = 1;- /** Creates {@code Lucene90PostingsFormat} with default settings. */+ /** Creates read-only {@code Lucene90PostingsFormat}. */public Lucene90PostingsFormat() {- this(- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);- }-- /**- * Creates {@code Lucene90PostingsFormat} with custom values for {@code minBlockSize} and {@code- * maxBlockSize} passed to block terms dictionary.- *- * @see- * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)- */- public Lucene90PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {super("Lucene90");- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);- this.minTermBlockSize = minTermBlockSize;- this.maxTermBlockSize = maxTermBlockSize;}@Override@@ -403,20 +385,8 @@ public final class Lucene90PostingsFormat extends PostingsFormat {}@Override- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);- boolean success = false;- try {- FieldsConsumer ret =- new Lucene90BlockTreeTermsWriter(- state, postingsWriter, minTermBlockSize, maxTermBlockSize);- success = true;- return ret;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(postingsWriter);- }- }+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) {+ throw new UnsupportedOperationException("Old codecs may only be used for reading");}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsReader.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.javaindex 56f7bafd4fe..190fe5b1d1d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.java@@ -14,23 +14,23 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;-import static org.apache.lucene.codecs.lucene90.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.VERSION_START;+import static org.apache.lucene.backward_codecs.lucene90.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.VERSION_START;import java.io.IOException;import java.util.Arrays;+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;import org.apache.lucene.codecs.BlockTermState;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.Impacts;import org.apache.lucene.index.ImpactsEnum;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsWriter.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsWriter.javasimilarity index 95%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsWriter.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsWriter.javaindex f640d8d2c57..b745d4e1217 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PostingsWriter.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsWriter.java@@ -14,22 +14,22 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;-import static org.apache.lucene.codecs.lucene90.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;+import static org.apache.lucene.backward_codecs.lucene90.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.DOC_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.MAX_SKIP_LEVELS;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.PAY_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.POS_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.VERSION_CURRENT;import java.io.IOException;+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;import org.apache.lucene.codecs.BlockTermState;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.CompetitiveImpactAccumulator;import org.apache.lucene.codecs.PushPostingsWriterBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexFileNames;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90ScoreSkipReader.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90ScoreSkipReader.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90ScoreSkipReader.javaindex 44789a98334..504fd5294e1 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90ScoreSkipReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90ScoreSkipReader.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import java.util.AbstractList;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipReader.javasimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipReader.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipReader.javaindex da31bd75a80..cd2febadf8f 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipReader.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import java.util.Arrays;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipWriter.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipWriter.javasimilarity index 98%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipWriter.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipWriter.javaindex 0743de226c8..72319eff6a5 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90SkipWriter.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90SkipWriter.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import java.util.Arrays;@@ -92,8 +92,7 @@ final class Lucene90SkipWriter extends MultiLevelSkipListWriter {}}- public void setField(- boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {+ void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {this.fieldHasPositions = fieldHasPositions;this.fieldHasOffsets = fieldHasOffsets;this.fieldHasPayloads = fieldHasPayloads;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/PForUtil.javasimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/PForUtil.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/PForUtil.javaindex 211912142a4..69b91660d10 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/PForUtil.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/PForUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import java.util.Arrays;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.pysimilarity index 99%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.pyrename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.pyindex a468c590cf9..6c58ce1a9a0 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/gen_ForUtil.py@@ -40,7 +40,7 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.backward_codecs.lucene90;import java.io.IOException;import org.apache.lucene.store.DataInput;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.javaindex 9393d0fa5a1..a354dc0827c 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91Codec.java@@ -18,6 +18,7 @@ package org.apache.lucene.backward_codecs.lucene91;import java.util.Objects;import org.apache.lucene.backward_codecs.lucene90.Lucene90FieldInfosFormat;+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.CompoundFormat;@@ -37,7 +38,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.javaindex 21bc835b61e..6dc0e4edb59 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/package-info.java@@ -151,17 +151,17 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A- * dictionary containing all of the terms used in all of the indexed fields of all of the+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.+ * A dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For- * each term in the dictionary, the numbers of all the documents that contain that term, and- * the frequency of the term in that document, unless frequencies are omitted ({@link- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For- * each term in the dictionary, the positions that the term occurs in each document. Note that- * this will not exist if all fields in all documents omit position data.+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency+ * data}. For each term in the dictionary, the numbers of all the documents that contain that+ * term, and the frequency of the term in that document, unless frequencies are omitted+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity+ * data}. For each term in the dictionary, the positions that the term occurs in each+ * document. Note that this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For* each field in each document, a value is stored that is multiplied into the score for hits* on that field.@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.javaindex 28ba9923481..15a61fd0c0d 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/Lucene92Codec.java@@ -18,6 +18,7 @@ package org.apache.lucene.backward_codecs.lucene92;import java.util.Objects;import org.apache.lucene.backward_codecs.lucene90.Lucene90FieldInfosFormat;+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.CompoundFormat;@@ -37,7 +38,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.javaindex 2e85de218e0..c993b78e9f1 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene92/package-info.java@@ -151,17 +151,17 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A- * dictionary containing all of the terms used in all of the indexed fields of all of the+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.+ * A dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For- * each term in the dictionary, the numbers of all the documents that contain that term, and- * the frequency of the term in that document, unless frequencies are omitted ({@link- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For- * each term in the dictionary, the positions that the term occurs in each document. Note that- * this will not exist if all fields in all documents omit position data.+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency+ * data}. For each term in the dictionary, the numbers of all the documents that contain that+ * term, and the frequency of the term in that document, unless frequencies are omitted+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity+ * data}. For each term in the dictionary, the positions that the term occurs in each+ * document. Note that this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For* each field in each document, a value is stored that is multiplied into the score for hits* on that field.@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.javaindex 324ee042967..d8bc7413feb 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java@@ -17,6 +17,7 @@package org.apache.lucene.backward_codecs.lucene94;import java.util.Objects;+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.CompoundFormat;@@ -36,7 +37,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.javaindex ae2bffe0bbb..cd140076a82 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java@@ -151,17 +151,17 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A- * dictionary containing all of the terms used in all of the indexed fields of all of the+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.+ * A dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For- * each term in the dictionary, the numbers of all the documents that contain that term, and- * the frequency of the term in that document, unless frequencies are omitted ({@link- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For- * each term in the dictionary, the positions that the term occurs in each document. Note that- * this will not exist if all fields in all documents omit position data.+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency+ * data}. For each term in the dictionary, the numbers of all the documents that contain that+ * term, and the frequency of the term in that document, unless frequencies are omitted+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity+ * data}. For each term in the dictionary, the positions that the term occurs in each+ * document. Note that this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For* each field in each document, a value is stored that is multiplied into the score for hits* on that field.@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.javaindex 4e42ff09472..e47de701502 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/Lucene95Codec.java@@ -17,6 +17,7 @@package org.apache.lucene.backward_codecs.lucene95;import java.util.Objects;+import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90SegmentInfoFormat;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.CompoundFormat;@@ -36,7 +37,6 @@ import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.javaindex 4ad0f34364a..879704dd842 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene95/package-info.java@@ -151,17 +151,17 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A- * dictionary containing all of the terms used in all of the indexed fields of all of the+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term dictionary}.+ * A dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For- * each term in the dictionary, the numbers of all the documents that contain that term, and- * the frequency of the term in that document, unless frequencies are omitted ({@link- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For- * each term in the dictionary, the positions that the term occurs in each document. Note that- * this will not exist if all fields in all documents omit position data.+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Frequency+ * data}. For each term in the dictionary, the numbers of all the documents that contain that+ * term, and the frequency of the term in that document, unless frequencies are omitted+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})+ * <li>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Proximity+ * data}. For each term in the dictionary, the positions that the term occurs in each+ * document. Note that this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For* each field in each document, a value is stored that is multiplied into the score for hits* on that field.@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex 620f7c81a9d..18ac359d2ae 100644--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -15,3 +15,4 @@org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormatorg.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat+org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormatdiff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.javaindex 7c032f7f58b..240a53f22db 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWCodec.java@@ -17,6 +17,7 @@package org.apache.lucene.backward_codecs.lucene90;import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -32,6 +33,7 @@ public class Lucene90RWCodec extends Lucene90Codec {return defaultKnnVectorsFormat;}};+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();public Lucene90RWCodec() {this.defaultKnnVectorsFormat =@@ -49,4 +51,9 @@ public class Lucene90RWCodec extends Lucene90Codec {public SegmentInfoFormat segmentInfoFormat() {return segmentInfosFormat;}++ @Override+ public PostingsFormat getPostingsFormatForField(String field) {+ return postingsFormat;+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.javanew file mode 100644index 00000000000..6497bb13d11--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java@@ -0,0 +1,184 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene90;++import java.io.IOException;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.FieldsConsumer;+import org.apache.lucene.codecs.FieldsProducer;+import org.apache.lucene.codecs.PostingsFormat;+import org.apache.lucene.codecs.PostingsReaderBase;+import org.apache.lucene.codecs.PostingsWriterBase;+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.index.TermState;+import org.apache.lucene.util.IOUtils;++public final class Lucene90RWPostingsFormat extends PostingsFormat {++ /** Size of blocks. */+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;++ /**+ * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,+ * but slower skipping in big posting lists.+ */+ static final int MAX_SKIP_LEVELS = 10;++ static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";+ static final String DOC_CODEC = "Lucene90PostingsWriterDoc";+ static final String POS_CODEC = "Lucene90PostingsWriterPos";+ static final String PAY_CODEC = "Lucene90PostingsWriterPay";++ private final int minTermBlockSize;+ private final int maxTermBlockSize;++ /** Creates {@code Lucene90RWPostingsFormat} with default settings. */+ public Lucene90RWPostingsFormat() {+ this(+ Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,+ Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);+ }++ public Lucene90RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {+ super("Lucene90");+ Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);+ this.minTermBlockSize = minTermBlockSize;+ this.maxTermBlockSize = maxTermBlockSize;+ }++ @Override+ public String toString() {+ return getName();+ }++ @Override+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {+ PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);+ boolean success = false;+ try {+ FieldsConsumer ret =+ new Lucene90BlockTreeTermsWriter(+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);+ success = true;+ return ret;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(postingsWriter);+ }+ }+ }++ @Override+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {+ PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);+ boolean success = false;+ try {+ FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);+ success = true;+ return ret;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(postingsReader);+ }+ }+ }++ /**+ * Holds all state required for {@link Lucene90PostingsReader} to produce a {@link+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.+ *+ * @lucene.internal+ */+ public static final class IntBlockTermState extends BlockTermState {+ /** file pointer to the start of the doc ids enumeration, in .doc file */+ public long docStartFP;++ /** file pointer to the start of the positions enumeration, in .pos file */+ public long posStartFP;++ /** file pointer to the start of the payloads enumeration, in .pay file */+ public long payStartFP;++ /**+ * file offset for the start of the skip list, relative to docStartFP, if there are more than+ * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1+ */+ public long skipOffset;++ /**+ * file offset for the last position in the last block, if there are more than {@link+ * ForUtil#BLOCK_SIZE} positions; otherwise -1+ *+ * <p>One might think to use total term frequency to track how many positions are left to read+ * as we decode the blocks, and decode the last block differently when num_left_positions <+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip+ * blocks as the skipper will only tell us new position offset (start of block) and number of+ * positions to skip for that block, without telling us how many positions it has skipped.+ */+ public long lastPosBlockOffset;++ /**+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly+ * totalTermFreq in this case.+ */+ public int singletonDocID;++ /** Sole constructor. */+ public IntBlockTermState() {+ skipOffset = -1;+ lastPosBlockOffset = -1;+ singletonDocID = -1;+ }++ @Override+ public IntBlockTermState clone() {+ IntBlockTermState other = new IntBlockTermState();+ other.copyFrom(this);+ return other;+ }++ @Override+ public void copyFrom(TermState _other) {+ super.copyFrom(_other);+ IntBlockTermState other = (IntBlockTermState) _other;+ docStartFP = other.docStartFP;+ posStartFP = other.posStartFP;+ payStartFP = other.payStartFP;+ lastPosBlockOffset = other.lastPosBlockOffset;+ skipOffset = other.skipOffset;+ singletonDocID = other.singletonDocID;+ }++ @Override+ public String toString() {+ return super.toString()+ + " docStartFP="+ + docStartFP+ + " posStartFP="+ + posStartFP+ + " payStartFP="+ + payStartFP+ + " lastPosBlockOffset="+ + lastPosBlockOffset+ + " singletonDocID="+ + singletonDocID;+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.javaindex f9bac90b906..ec775b513cd 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/Lucene91RWCodec.java@@ -16,8 +16,10 @@*/package org.apache.lucene.backward_codecs.lucene91;+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -32,6 +34,7 @@ public class Lucene91RWCodec extends Lucene91Codec {}};private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();public Lucene91RWCodec() {this.defaultKnnVectorsFormat =@@ -49,4 +52,9 @@ public class Lucene91RWCodec extends Lucene91Codec {public SegmentInfoFormat segmentInfoFormat() {return segmentInfosFormat;}++ @Override+ public PostingsFormat getPostingsFormatForField(String field) {+ return postingsFormat;+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.javaindex 6008fa5df39..8d086787413 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/Lucene92RWCodec.java@@ -16,8 +16,10 @@*/package org.apache.lucene.backward_codecs.lucene92;+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -33,6 +35,7 @@ public class Lucene92RWCodec extends Lucene92Codec {}};private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();/** Instantiates a new codec. */public Lucene92RWCodec() {@@ -51,4 +54,9 @@ public class Lucene92RWCodec extends Lucene92Codec {public SegmentInfoFormat segmentInfoFormat() {return segmentInfosFormat;}++ @Override+ public PostingsFormat getPostingsFormatForField(String field) {+ return postingsFormat;+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.javaindex 88edd2431f8..678dec4ad19 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/Lucene94RWCodec.java@@ -16,8 +16,10 @@*/package org.apache.lucene.backward_codecs.lucene94;+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -32,8 +34,8 @@ public class Lucene94RWCodec extends Lucene94Codec {return defaultKnnVectorsFormat;}};-private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();/** Instantiates a new codec. */public Lucene94RWCodec() {@@ -52,4 +54,9 @@ public class Lucene94RWCodec extends Lucene94Codec {public SegmentInfoFormat segmentInfoFormat() {return segmentInfosFormat;}++ @Override+ public PostingsFormat getPostingsFormatForField(String field) {+ return postingsFormat;+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.javaindex c8311dfd115..e51a2a391dd 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene95/Lucene95RWCodec.java@@ -16,9 +16,11 @@*/package org.apache.lucene.backward_codecs.lucene95;+import org.apache.lucene.backward_codecs.lucene90.Lucene90RWPostingsFormat;import org.apache.lucene.backward_codecs.lucene90.Lucene90RWSegmentInfoFormat;import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat;import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;@@ -33,6 +35,7 @@ public class Lucene95RWCodec extends Lucene95Codec {return defaultKnnVectorsFormat;}};+ private final PostingsFormat postingsFormat = new Lucene90RWPostingsFormat();private final SegmentInfoFormat segmentInfosFormat = new Lucene90RWSegmentInfoFormat();/** Instantiates a new codec. */@@ -52,4 +55,9 @@ public class Lucene95RWCodec extends Lucene95Codec {public SegmentInfoFormat segmentInfoFormat() {return segmentInfosFormat;}++ @Override+ public PostingsFormat getPostingsFormatForField(String field) {+ return postingsFormat;+ }}diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.javaindex 5ee3451c9e8..d389372d8f8 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java@@ -22,14 +22,14 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.util.IOUtils;-/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene90PostingsWriter}. */+/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene99PostingsWriter}. */public class BlockTreeOrdsPostingsFormat extends PostingsFormat {private final int minTermBlockSize;@@ -67,7 +67,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);boolean success = false;try {@@ -84,7 +84,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);boolean success = false;try {FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.javaindex c243dd0c0ee..4c355f314f1 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java@@ -24,7 +24,7 @@ import java.util.TreeMap;import org.apache.lucene.codecs.FieldsConsumer;import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;import org.apache.lucene.index.BaseTermsEnum;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.Fields;@@ -54,7 +54,7 @@ import org.apache.lucene.util.automaton.TransitionAccessor;// - or: longer dense skip lists than just next byte?/**- * Wraps {@link Lucene90PostingsFormat} format for on-disk storage, but then at read time loads and+ * Wraps {@link Lucene99PostingsFormat} format for on-disk storage, but then at read time loads and* stores all terms and postings directly in RAM as byte[], int[].** <p><b>WARNING</b>: This is exceptionally RAM intensive: it makes no effort to compress the@@ -97,12 +97,12 @@ public final class DirectPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- return PostingsFormat.forName("Lucene90").fieldsConsumer(state);+ return PostingsFormat.forName("Lucene99").fieldsConsumer(state);}@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- FieldsProducer postings = PostingsFormat.forName("Lucene90").fieldsProducer(state);+ FieldsProducer postings = PostingsFormat.forName("Lucene99").fieldsProducer(state);if (state.context.context != IOContext.Context.MERGE) {FieldsProducer loadedPostings;try {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.javaindex 4e0fec1587f..0a18b75ac00 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java@@ -22,8 +22,8 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.util.IOUtils;@@ -41,7 +41,7 @@ public final class FSTPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);boolean success = false;try {@@ -57,7 +57,7 @@ public final class FSTPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);boolean success = false;try {FieldsProducer ret = new FSTTermsReader(state, postingsReader);diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.javaindex 6881aedbd50..ae01216ccf6 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java@@ -17,13 +17,13 @@package org.apache.lucene.codecs.uniformsplit;-import static org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.BLOCK_SIZE;import java.io.IOException;import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.TermState;@@ -34,7 +34,7 @@ import org.apache.lucene.util.RamUsageEstimator;/*** {@link TermState} serializer which encodes each file pointer as a delta relative to a base file- * pointer. It differs from {@link Lucene90PostingsWriter#encodeTerm} which encodes each file+ * pointer. It differs from {@link Lucene99PostingsWriter#encodeTerm} which encodes each file* pointer as a delta relative to the previous file pointer.** <p>It automatically sets the base file pointer to the first valid file pointer for doc start FP,@@ -95,7 +95,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {/*** Writes a {@link BlockTermState} to the provided {@link DataOutput}.*- * <p>Simpler variant of {@link Lucene90PostingsWriter#encodeTerm(DataOutput, FieldInfo,+ * <p>Simpler variant of {@link Lucene99PostingsWriter#encodeTerm(DataOutput, FieldInfo,* BlockTermState, boolean)}.*/public void writeTermState(@@ -148,7 +148,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {/*** Reads a {@link BlockTermState} from the provided {@link DataInput}.*- * <p>Simpler variant of {@link Lucene90PostingsReader#decodeTerm(DataInput, FieldInfo,+ * <p>Simpler variant of {@link Lucene99PostingsReader#decodeTerm(DataInput, FieldInfo,* BlockTermState, boolean)}.** @param reuse {@link BlockTermState} to reuse; or null to create a new one.diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.javaindex 5ac38e78b8f..8e2b7bb7d54 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.util.IOUtils;@@ -113,7 +113,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);boolean success = false;try {FieldsConsumer termsWriter =@@ -130,7 +130,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);boolean success = false;try {FieldsProducer termsReader =diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.javaindex f1df01d09b2..a0fa67508f3 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java@@ -28,7 +28,7 @@* org.apache.lucene.search.PhraseQuery})* <li>Quite efficient for {@link org.apache.lucene.search.PrefixQuery}* <li>Not efficient for spell-check and {@link org.apache.lucene.search.FuzzyQuery}, in this case- * prefer {@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat}+ * prefer {@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}* </ul>*/package org.apache.lucene.codecs.uniformsplit;diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.javaindex 69c36e792c0..dde59993f6e 100644--- a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java@@ -17,7 +17,7 @@package org.apache.lucene.codecs.lucene90.tests;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat.IntBlockTermState;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;/** Test utility class to create mock {@link IntBlockTermState}. */public class MockTermStateFactory {diff --git a/lucene/core/src/generated/checksums/generateForUtil.json b/lucene/core/src/generated/checksums/generateForUtil.jsonindex 6cd91640bd1..527070d7bf0 100644--- a/lucene/core/src/generated/checksums/generateForUtil.json+++ b/lucene/core/src/generated/checksums/generateForUtil.json@@ -1,4 +1,4 @@{- "lucene/core/src/java/org/apache/lucene/codecs/lucene90/ForUtil.java": "a67f47001bad680d75c57a864a1552a78fc85600",- "lucene/core/src/java/org/apache/lucene/codecs/lucene90/gen_ForUtil.py": "3219c6d0289cc64d80dbdbc5db4bd05b8774e797"+ "lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java": "1292ad354d255b1272ffd3db684aa2ddb2bc49ec",+ "lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py": "ab7b63a1b73986cc04e43de1c8f474b97aef5116"}\ No newline at end of filediff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.javaindex 98210a89300..096c965dc98 100644--- a/lucene/core/src/java/module-info.java+++ b/lucene/core/src/java/module-info.java@@ -72,7 +72,7 @@ module org.apache.lucene.core {provides org.apache.lucene.codecs.KnnVectorsFormat withLucene99HnswVectorsFormat;provides org.apache.lucene.codecs.PostingsFormat with- org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;+ org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;provides org.apache.lucene.index.SortFieldProvider withorg.apache.lucene.search.SortField.Provider,org.apache.lucene.search.SortedNumericSortField.Provider,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.javanew file mode 100644index 00000000000..2403c997435--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java@@ -0,0 +1,78 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.packed.PackedInts;++/** Utility class to encode/decode increasing sequences of 128 integers. */+public class ForDeltaUtil {++ // IDENTITY_PLUS_ONE[i] == i+1+ private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];++ static {+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ IDENTITY_PLUS_ONE[i] = i + 1;+ }+ }++ private static void prefixSumOfOnes(long[] arr, long base) {+ System.arraycopy(IDENTITY_PLUS_ONE, 0, arr, 0, ForUtil.BLOCK_SIZE);+ // This loop gets auto-vectorized+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ arr[i] += base;+ }+ }++ private final ForUtil forUtil;++ ForDeltaUtil(ForUtil forUtil) {+ this.forUtil = forUtil;+ }++ /**+ * Encode deltas of a strictly monotonically increasing sequence of integers. The provided {@code+ * longs} are expected to be deltas between consecutive values.+ */+ void encodeDeltas(long[] longs, DataOutput out) throws IOException {+ if (longs[0] == 1 && PForUtil.allEqual(longs)) { // happens with very dense postings+ out.writeByte((byte) 0);+ } else {+ long or = 0;+ for (long l : longs) {+ or |= l;+ }+ assert or != 0;+ final int bitsPerValue = PackedInts.bitsRequired(or);+ out.writeByte((byte) bitsPerValue);+ forUtil.encode(longs, bitsPerValue, out);+ }+ }++ /** Decode deltas, compute the prefix sum and add {@code base} to all decoded longs. */+ void decodeAndPrefixSum(DataInput in, long base, long[] longs) throws IOException {+ final int bitsPerValue = Byte.toUnsignedInt(in.readByte());+ if (bitsPerValue == 0) {+ prefixSumOfOnes(longs, base);+ } else {+ forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.javanew file mode 100644index 00000000000..3126cdb02c0--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java@@ -0,0 +1,1148 @@+// This file has been automatically generated, DO NOT EDIT++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.DataOutput;++// Inspired from https://fulmicoton.com/posts/bitpacking/+// Encodes multiple integers in a long to get SIMD-like speedups.+// If bitsPerValue <= 8 then we pack 8 ints per long+// else if bitsPerValue <= 16 we pack 4 ints per long+// else we pack 2 ints per long+final class ForUtil {++ static final int BLOCK_SIZE = 128;+ private static final int BLOCK_SIZE_LOG2 = 7;++ private static long expandMask32(long mask32) {+ return mask32 | (mask32 << 32);+ }++ private static long expandMask16(long mask16) {+ return expandMask32(mask16 | (mask16 << 16));+ }++ private static long expandMask8(long mask8) {+ return expandMask16(mask8 | (mask8 << 8));+ }++ private static long mask32(int bitsPerValue) {+ return expandMask32((1L << bitsPerValue) - 1);+ }++ private static long mask16(int bitsPerValue) {+ return expandMask16((1L << bitsPerValue) - 1);+ }++ private static long mask8(int bitsPerValue) {+ return expandMask8((1L << bitsPerValue) - 1);+ }++ private static void expand8(long[] arr) {+ for (int i = 0; i < 16; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 56) & 0xFFL;+ arr[16 + i] = (l >>> 48) & 0xFFL;+ arr[32 + i] = (l >>> 40) & 0xFFL;+ arr[48 + i] = (l >>> 32) & 0xFFL;+ arr[64 + i] = (l >>> 24) & 0xFFL;+ arr[80 + i] = (l >>> 16) & 0xFFL;+ arr[96 + i] = (l >>> 8) & 0xFFL;+ arr[112 + i] = l & 0xFFL;+ }+ }++ private static void expand8To32(long[] arr) {+ for (int i = 0; i < 16; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 24) & 0x000000FF000000FFL;+ arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;+ arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;+ arr[48 + i] = l & 0x000000FF000000FFL;+ }+ }++ private static void collapse8(long[] arr) {+ for (int i = 0; i < 16; ++i) {+ arr[i] =+ (arr[i] << 56)+ | (arr[16 + i] << 48)+ | (arr[32 + i] << 40)+ | (arr[48 + i] << 32)+ | (arr[64 + i] << 24)+ | (arr[80 + i] << 16)+ | (arr[96 + i] << 8)+ | arr[112 + i];+ }+ }++ private static void expand16(long[] arr) {+ for (int i = 0; i < 32; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 48) & 0xFFFFL;+ arr[32 + i] = (l >>> 32) & 0xFFFFL;+ arr[64 + i] = (l >>> 16) & 0xFFFFL;+ arr[96 + i] = l & 0xFFFFL;+ }+ }++ private static void expand16To32(long[] arr) {+ for (int i = 0; i < 32; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;+ arr[32 + i] = l & 0x0000FFFF0000FFFFL;+ }+ }++ private static void collapse16(long[] arr) {+ for (int i = 0; i < 32; ++i) {+ arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];+ }+ }++ private static void expand32(long[] arr) {+ for (int i = 0; i < 64; ++i) {+ long l = arr[i];+ arr[i] = l >>> 32;+ arr[64 + i] = l & 0xFFFFFFFFL;+ }+ }++ private static void collapse32(long[] arr) {+ for (int i = 0; i < 64; ++i) {+ arr[i] = (arr[i] << 32) | arr[64 + i];+ }+ }++ private static void prefixSum8(long[] arr, long base) {+ expand8To32(arr);+ prefixSum32(arr, base);+ }++ private static void prefixSum16(long[] arr, long base) {+ // We need to move to the next primitive size to avoid overflows+ expand16To32(arr);+ prefixSum32(arr, base);+ }++ private static void prefixSum32(long[] arr, long base) {+ arr[0] += base << 32;+ innerPrefixSum32(arr);+ expand32(arr);+ final long l = arr[BLOCK_SIZE / 2 - 1];+ for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) {+ arr[i] += l;+ }+ }++ // For some reason unrolling seems to help+ private static void innerPrefixSum32(long[] arr) {+ arr[1] += arr[0];+ arr[2] += arr[1];+ arr[3] += arr[2];+ arr[4] += arr[3];+ arr[5] += arr[4];+ arr[6] += arr[5];+ arr[7] += arr[6];+ arr[8] += arr[7];+ arr[9] += arr[8];+ arr[10] += arr[9];+ arr[11] += arr[10];+ arr[12] += arr[11];+ arr[13] += arr[12];+ arr[14] += arr[13];+ arr[15] += arr[14];+ arr[16] += arr[15];+ arr[17] += arr[16];+ arr[18] += arr[17];+ arr[19] += arr[18];+ arr[20] += arr[19];+ arr[21] += arr[20];+ arr[22] += arr[21];+ arr[23] += arr[22];+ arr[24] += arr[23];+ arr[25] += arr[24];+ arr[26] += arr[25];+ arr[27] += arr[26];+ arr[28] += arr[27];+ arr[29] += arr[28];+ arr[30] += arr[29];+ arr[31] += arr[30];+ arr[32] += arr[31];+ arr[33] += arr[32];+ arr[34] += arr[33];+ arr[35] += arr[34];+ arr[36] += arr[35];+ arr[37] += arr[36];+ arr[38] += arr[37];+ arr[39] += arr[38];+ arr[40] += arr[39];+ arr[41] += arr[40];+ arr[42] += arr[41];+ arr[43] += arr[42];+ arr[44] += arr[43];+ arr[45] += arr[44];+ arr[46] += arr[45];+ arr[47] += arr[46];+ arr[48] += arr[47];+ arr[49] += arr[48];+ arr[50] += arr[49];+ arr[51] += arr[50];+ arr[52] += arr[51];+ arr[53] += arr[52];+ arr[54] += arr[53];+ arr[55] += arr[54];+ arr[56] += arr[55];+ arr[57] += arr[56];+ arr[58] += arr[57];+ arr[59] += arr[58];+ arr[60] += arr[59];+ arr[61] += arr[60];+ arr[62] += arr[61];+ arr[63] += arr[62];+ }++ private final long[] tmp = new long[BLOCK_SIZE / 2];++ /** Encode 128 integers from {@code longs} into {@code out}. */+ void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {+ final int nextPrimitive;+ final int numLongs;+ if (bitsPerValue <= 8) {+ nextPrimitive = 8;+ numLongs = BLOCK_SIZE / 8;+ collapse8(longs);+ } else if (bitsPerValue <= 16) {+ nextPrimitive = 16;+ numLongs = BLOCK_SIZE / 4;+ collapse16(longs);+ } else {+ nextPrimitive = 32;+ numLongs = BLOCK_SIZE / 2;+ collapse32(longs);+ }++ final int numLongsPerShift = bitsPerValue * 2;+ int idx = 0;+ int shift = nextPrimitive - bitsPerValue;+ for (int i = 0; i < numLongsPerShift; ++i) {+ tmp[i] = longs[idx++] << shift;+ }+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {+ for (int i = 0; i < numLongsPerShift; ++i) {+ tmp[i] |= longs[idx++] << shift;+ }+ }++ final int remainingBitsPerLong = shift + bitsPerValue;+ final long maskRemainingBitsPerLong;+ if (nextPrimitive == 8) {+ maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];+ } else if (nextPrimitive == 16) {+ maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];+ } else {+ maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];+ }++ int tmpIdx = 0;+ int remainingBitsPerValue = bitsPerValue;+ while (idx < numLongs) {+ if (remainingBitsPerValue >= remainingBitsPerLong) {+ remainingBitsPerValue -= remainingBitsPerLong;+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;+ if (remainingBitsPerValue == 0) {+ idx++;+ remainingBitsPerValue = bitsPerValue;+ }+ } else {+ final long mask1, mask2;+ if (nextPrimitive == 8) {+ mask1 = MASKS8[remainingBitsPerValue];+ mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];+ } else if (nextPrimitive == 16) {+ mask1 = MASKS16[remainingBitsPerValue];+ mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];+ } else {+ mask1 = MASKS32[remainingBitsPerValue];+ mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];+ }+ tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);+ remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;+ }+ }++ for (int i = 0; i < numLongsPerShift; ++i) {+ out.writeLong(tmp[i]);+ }+ }++ /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */+ int numBytes(int bitsPerValue) {+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);+ }++ private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)+ throws IOException {+ final int numLongs = bitsPerValue << 1;+ in.readLongs(tmp, 0, numLongs);+ final long mask = MASKS32[bitsPerValue];+ int longsIdx = 0;+ int shift = 32 - bitsPerValue;+ for (; shift >= 0; shift -= bitsPerValue) {+ shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);+ longsIdx += numLongs;+ }+ final int remainingBitsPerLong = shift + bitsPerValue;+ final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];+ int tmpIdx = 0;+ int remainingBits = remainingBitsPerLong;+ for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {+ int b = bitsPerValue - remainingBits;+ long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;+ while (b >= remainingBitsPerLong) {+ b -= remainingBitsPerLong;+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;+ }+ if (b > 0) {+ l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];+ remainingBits = remainingBitsPerLong - b;+ } else {+ remainingBits = remainingBitsPerLong;+ }+ longs[longsIdx] = l;+ }+ }++ /**+ * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which+ * generates SIMD instructions for it in order to shift multiple longs at once.+ */+ private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {+ for (int i = 0; i < count; ++i) {+ b[bi + i] = (a[i] >>> shift) & mask;+ }+ }++ private static final long[] MASKS8 = new long[8];+ private static final long[] MASKS16 = new long[16];+ private static final long[] MASKS32 = new long[32];++ static {+ for (int i = 0; i < 8; ++i) {+ MASKS8[i] = mask8(i);+ }+ for (int i = 0; i < 16; ++i) {+ MASKS16[i] = mask16(i);+ }+ for (int i = 0; i < 32; ++i) {+ MASKS32[i] = mask32(i);+ }+ }++ // mark values in array as final longs to avoid the cost of reading array, arrays should only be+ // used when the idx is a variable+ private static final long MASK8_1 = MASKS8[1];+ private static final long MASK8_2 = MASKS8[2];+ private static final long MASK8_3 = MASKS8[3];+ private static final long MASK8_4 = MASKS8[4];+ private static final long MASK8_5 = MASKS8[5];+ private static final long MASK8_6 = MASKS8[6];+ private static final long MASK8_7 = MASKS8[7];+ private static final long MASK16_1 = MASKS16[1];+ private static final long MASK16_2 = MASKS16[2];+ private static final long MASK16_3 = MASKS16[3];+ private static final long MASK16_4 = MASKS16[4];+ private static final long MASK16_5 = MASKS16[5];+ private static final long MASK16_6 = MASKS16[6];+ private static final long MASK16_7 = MASKS16[7];+ private static final long MASK16_9 = MASKS16[9];+ private static final long MASK16_10 = MASKS16[10];+ private static final long MASK16_11 = MASKS16[11];+ private static final long MASK16_12 = MASKS16[12];+ private static final long MASK16_13 = MASKS16[13];+ private static final long MASK16_14 = MASKS16[14];+ private static final long MASK16_15 = MASKS16[15];+ private static final long MASK32_1 = MASKS32[1];+ private static final long MASK32_2 = MASKS32[2];+ private static final long MASK32_3 = MASKS32[3];+ private static final long MASK32_4 = MASKS32[4];+ private static final long MASK32_5 = MASKS32[5];+ private static final long MASK32_6 = MASKS32[6];+ private static final long MASK32_7 = MASKS32[7];+ private static final long MASK32_8 = MASKS32[8];+ private static final long MASK32_9 = MASKS32[9];+ private static final long MASK32_10 = MASKS32[10];+ private static final long MASK32_11 = MASKS32[11];+ private static final long MASK32_12 = MASKS32[12];+ private static final long MASK32_13 = MASKS32[13];+ private static final long MASK32_14 = MASKS32[14];+ private static final long MASK32_15 = MASKS32[15];+ private static final long MASK32_17 = MASKS32[17];+ private static final long MASK32_18 = MASKS32[18];+ private static final long MASK32_19 = MASKS32[19];+ private static final long MASK32_20 = MASKS32[20];+ private static final long MASK32_21 = MASKS32[21];+ private static final long MASK32_22 = MASKS32[22];+ private static final long MASK32_23 = MASKS32[23];+ private static final long MASK32_24 = MASKS32[24];++ /** Decode 128 integers into {@code longs}. */+ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {+ switch (bitsPerValue) {+ case 1:+ decode1(in, tmp, longs);+ expand8(longs);+ break;+ case 2:+ decode2(in, tmp, longs);+ expand8(longs);+ break;+ case 3:+ decode3(in, tmp, longs);+ expand8(longs);+ break;+ case 4:+ decode4(in, tmp, longs);+ expand8(longs);+ break;+ case 5:+ decode5(in, tmp, longs);+ expand8(longs);+ break;+ case 6:+ decode6(in, tmp, longs);+ expand8(longs);+ break;+ case 7:+ decode7(in, tmp, longs);+ expand8(longs);+ break;+ case 8:+ decode8(in, tmp, longs);+ expand8(longs);+ break;+ case 9:+ decode9(in, tmp, longs);+ expand16(longs);+ break;+ case 10:+ decode10(in, tmp, longs);+ expand16(longs);+ break;+ case 11:+ decode11(in, tmp, longs);+ expand16(longs);+ break;+ case 12:+ decode12(in, tmp, longs);+ expand16(longs);+ break;+ case 13:+ decode13(in, tmp, longs);+ expand16(longs);+ break;+ case 14:+ decode14(in, tmp, longs);+ expand16(longs);+ break;+ case 15:+ decode15(in, tmp, longs);+ expand16(longs);+ break;+ case 16:+ decode16(in, tmp, longs);+ expand16(longs);+ break;+ case 17:+ decode17(in, tmp, longs);+ expand32(longs);+ break;+ case 18:+ decode18(in, tmp, longs);+ expand32(longs);+ break;+ case 19:+ decode19(in, tmp, longs);+ expand32(longs);+ break;+ case 20:+ decode20(in, tmp, longs);+ expand32(longs);+ break;+ case 21:+ decode21(in, tmp, longs);+ expand32(longs);+ break;+ case 22:+ decode22(in, tmp, longs);+ expand32(longs);+ break;+ case 23:+ decode23(in, tmp, longs);+ expand32(longs);+ break;+ case 24:+ decode24(in, tmp, longs);+ expand32(longs);+ break;+ default:+ decodeSlow(bitsPerValue, in, tmp, longs);+ expand32(longs);+ break;+ }+ }++ /** Delta-decode 128 integers into {@code longs}. */+ void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs)+ throws IOException {+ switch (bitsPerValue) {+ case 1:+ decode1(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 2:+ decode2(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 3:+ decode3(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 4:+ decode4(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 5:+ decode5(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 6:+ decode6(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 7:+ decode7(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 8:+ decode8(in, tmp, longs);+ prefixSum8(longs, base);+ break;+ case 9:+ decode9(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 10:+ decode10(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 11:+ decode11(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 12:+ decode12(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 13:+ decode13(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 14:+ decode14(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 15:+ decode15(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 16:+ decode16(in, tmp, longs);+ prefixSum16(longs, base);+ break;+ case 17:+ decode17(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 18:+ decode18(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 19:+ decode19(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 20:+ decode20(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 21:+ decode21(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 22:+ decode22(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 23:+ decode23(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ case 24:+ decode24(in, tmp, longs);+ prefixSum32(longs, base);+ break;+ default:+ decodeSlow(bitsPerValue, in, tmp, longs);+ prefixSum32(longs, base);+ break;+ }+ }++ private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 2);+ shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);+ shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);+ shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);+ shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);+ shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);+ shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);+ shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);+ shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);+ }++ private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 4);+ shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);+ shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);+ shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);+ shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);+ }++ private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 6);+ shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);+ shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);+ for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {+ long l0 = (tmp[tmpIdx + 0] & MASK8_2) << 1;+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;+ l1 |= (tmp[tmpIdx + 2] & MASK8_2) << 0;+ longs[longsIdx + 1] = l1;+ }+ }++ private static void decode4(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 8);+ shiftLongs(tmp, 8, longs, 0, 4, MASK8_4);+ shiftLongs(tmp, 8, longs, 8, 0, MASK8_4);+ }++ private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 10);+ shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);+ for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {+ long l0 = (tmp[tmpIdx + 0] & MASK8_3) << 2;+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;+ l1 |= (tmp[tmpIdx + 2] & MASK8_3) << 1;+ l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;+ l2 |= (tmp[tmpIdx + 4] & MASK8_3) << 0;+ longs[longsIdx + 2] = l2;+ }+ }++ private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 12);+ shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);+ shiftLongs(tmp, 12, tmp, 0, 0, MASK8_2);+ for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {+ long l0 = tmp[tmpIdx + 0] << 4;+ l0 |= tmp[tmpIdx + 1] << 2;+ l0 |= tmp[tmpIdx + 2] << 0;+ longs[longsIdx + 0] = l0;+ }+ }++ private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 14);+ shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);+ shiftLongs(tmp, 14, tmp, 0, 0, MASK8_1);+ for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {+ long l0 = tmp[tmpIdx + 0] << 6;+ l0 |= tmp[tmpIdx + 1] << 5;+ l0 |= tmp[tmpIdx + 2] << 4;+ l0 |= tmp[tmpIdx + 3] << 3;+ l0 |= tmp[tmpIdx + 4] << 2;+ l0 |= tmp[tmpIdx + 5] << 1;+ l0 |= tmp[tmpIdx + 6] << 0;+ longs[longsIdx + 0] = l0;+ }+ }++ private static void decode8(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(longs, 0, 16);+ }++ private static void decode9(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 18);+ shiftLongs(tmp, 18, longs, 0, 7, MASK16_9);+ for (int iter = 0, tmpIdx = 0, longsIdx = 18; iter < 2; ++iter, tmpIdx += 9, longsIdx += 7) {+ long l0 = (tmp[tmpIdx + 0] & MASK16_7) << 2;+ l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;+ l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;+ l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;+ l3 |= (tmp[tmpIdx + 4] & MASK16_7) << 1;+ l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;+ l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;+ longs[longsIdx + 4] = l4;+ long l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;+ l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;+ longs[longsIdx + 5] = l5;+ long l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;+ l6 |= (tmp[tmpIdx + 8] & MASK16_7) << 0;+ longs[longsIdx + 6] = l6;+ }+ }++ private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 20);+ shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);+ for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {+ long l0 = (tmp[tmpIdx + 0] & MASK16_6) << 4;+ l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;+ l1 |= (tmp[tmpIdx + 2] & MASK16_6) << 2;+ l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;+ l2 |= (tmp[tmpIdx + 4] & MASK16_6) << 0;+ longs[longsIdx + 2] = l2;+ }+ }++ private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 22);+ shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);+ for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {+ long l0 = (tmp[tmpIdx + 0] & MASK16_5) << 6;+ l0 |= (tmp[tmpIdx + 1] & MASK16_5) << 1;+ l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;+ l1 |= (tmp[tmpIdx + 3] & MASK16_5) << 2;+ l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;+ l2 |= (tmp[tmpIdx + 5] & MASK16_5) << 3;+ l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;+ l3 |= (tmp[tmpIdx + 7] & MASK16_5) << 4;+ l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;+ l4 |= (tmp[tmpIdx + 9] & MASK16_5) << 5;+ l4 |= (tmp[tmpIdx + 10] & MASK16_5) << 0;+ longs[longsIdx + 4] = l4;+ }+ }++ private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 24);+ shiftLongs(tmp, 24, longs, 0, 4, MASK16_12);+ shiftLongs(tmp, 24, tmp, 0, 0, MASK16_4);+ for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) {+ long l0 = tmp[tmpIdx + 0] << 8;+ l0 |= tmp[tmpIdx + 1] << 4;+ l0 |= tmp[tmpIdx + 2] << 0;+ longs[longsIdx + 0] = l0;+ }+ }++ private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 26);+ shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);+ for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {+ long l0 = (tmp[tmpIdx + 0] & MASK16_3) << 10;+ l0 |= (tmp[tmpIdx + 1] & MASK16_3) << 7;+ l0 |= (tmp[tmpIdx + 2] & MASK16_3) << 4;+ l0 |= (tmp[tmpIdx + 3] & MASK16_3) << 1;+ l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;+ l1 |= (tmp[tmpIdx + 5] & MASK16_3) << 8;+ l1 |= (tmp[tmpIdx + 6] & MASK16_3) << 5;+ l1 |= (tmp[tmpIdx + 7] & MASK16_3) << 2;+ l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;+ l2 |= (tmp[tmpIdx + 9] & MASK16_3) << 9;+ l2 |= (tmp[tmpIdx + 10] & MASK16_3) << 6;+ l2 |= (tmp[tmpIdx + 11] & MASK16_3) << 3;+ l2 |= (tmp[tmpIdx + 12] & MASK16_3) << 0;+ longs[longsIdx + 2] = l2;+ }+ }++ private static void decode14(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 28);+ shiftLongs(tmp, 28, longs, 0, 2, MASK16_14);+ shiftLongs(tmp, 28, tmp, 0, 0, MASK16_2);+ for (int iter = 0, tmpIdx = 0, longsIdx = 28; iter < 4; ++iter, tmpIdx += 7, longsIdx += 1) {+ long l0 = tmp[tmpIdx + 0] << 12;+ l0 |= tmp[tmpIdx + 1] << 10;+ l0 |= tmp[tmpIdx + 2] << 8;+ l0 |= tmp[tmpIdx + 3] << 6;+ l0 |= tmp[tmpIdx + 4] << 4;+ l0 |= tmp[tmpIdx + 5] << 2;+ l0 |= tmp[tmpIdx + 6] << 0;+ longs[longsIdx + 0] = l0;+ }+ }++ private static void decode15(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 30);+ shiftLongs(tmp, 30, longs, 0, 1, MASK16_15);+ shiftLongs(tmp, 30, tmp, 0, 0, MASK16_1);+ for (int iter = 0, tmpIdx = 0, longsIdx = 30; iter < 2; ++iter, tmpIdx += 15, longsIdx += 1) {+ long l0 = tmp[tmpIdx + 0] << 14;+ l0 |= tmp[tmpIdx + 1] << 13;+ l0 |= tmp[tmpIdx + 2] << 12;+ l0 |= tmp[tmpIdx + 3] << 11;+ l0 |= tmp[tmpIdx + 4] << 10;+ l0 |= tmp[tmpIdx + 5] << 9;+ l0 |= tmp[tmpIdx + 6] << 8;+ l0 |= tmp[tmpIdx + 7] << 7;+ l0 |= tmp[tmpIdx + 8] << 6;+ l0 |= tmp[tmpIdx + 9] << 5;+ l0 |= tmp[tmpIdx + 10] << 4;+ l0 |= tmp[tmpIdx + 11] << 3;+ l0 |= tmp[tmpIdx + 12] << 2;+ l0 |= tmp[tmpIdx + 13] << 1;+ l0 |= tmp[tmpIdx + 14] << 0;+ longs[longsIdx + 0] = l0;+ }+ }++ private static void decode16(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(longs, 0, 32);+ }++ private static void decode17(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 34);+ shiftLongs(tmp, 34, longs, 0, 15, MASK32_17);+ for (int iter = 0, tmpIdx = 0, longsIdx = 34; iter < 2; ++iter, tmpIdx += 17, longsIdx += 15) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_15) << 2;+ l0 |= (tmp[tmpIdx + 1] >>> 13) & MASK32_2;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK32_13) << 4;+ l1 |= (tmp[tmpIdx + 2] >>> 11) & MASK32_4;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 2] & MASK32_11) << 6;+ l2 |= (tmp[tmpIdx + 3] >>> 9) & MASK32_6;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 3] & MASK32_9) << 8;+ l3 |= (tmp[tmpIdx + 4] >>> 7) & MASK32_8;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 4] & MASK32_7) << 10;+ l4 |= (tmp[tmpIdx + 5] >>> 5) & MASK32_10;+ longs[longsIdx + 4] = l4;+ long l5 = (tmp[tmpIdx + 5] & MASK32_5) << 12;+ l5 |= (tmp[tmpIdx + 6] >>> 3) & MASK32_12;+ longs[longsIdx + 5] = l5;+ long l6 = (tmp[tmpIdx + 6] & MASK32_3) << 14;+ l6 |= (tmp[tmpIdx + 7] >>> 1) & MASK32_14;+ longs[longsIdx + 6] = l6;+ long l7 = (tmp[tmpIdx + 7] & MASK32_1) << 16;+ l7 |= (tmp[tmpIdx + 8] & MASK32_15) << 1;+ l7 |= (tmp[tmpIdx + 9] >>> 14) & MASK32_1;+ longs[longsIdx + 7] = l7;+ long l8 = (tmp[tmpIdx + 9] & MASK32_14) << 3;+ l8 |= (tmp[tmpIdx + 10] >>> 12) & MASK32_3;+ longs[longsIdx + 8] = l8;+ long l9 = (tmp[tmpIdx + 10] & MASK32_12) << 5;+ l9 |= (tmp[tmpIdx + 11] >>> 10) & MASK32_5;+ longs[longsIdx + 9] = l9;+ long l10 = (tmp[tmpIdx + 11] & MASK32_10) << 7;+ l10 |= (tmp[tmpIdx + 12] >>> 8) & MASK32_7;+ longs[longsIdx + 10] = l10;+ long l11 = (tmp[tmpIdx + 12] & MASK32_8) << 9;+ l11 |= (tmp[tmpIdx + 13] >>> 6) & MASK32_9;+ longs[longsIdx + 11] = l11;+ long l12 = (tmp[tmpIdx + 13] & MASK32_6) << 11;+ l12 |= (tmp[tmpIdx + 14] >>> 4) & MASK32_11;+ longs[longsIdx + 12] = l12;+ long l13 = (tmp[tmpIdx + 14] & MASK32_4) << 13;+ l13 |= (tmp[tmpIdx + 15] >>> 2) & MASK32_13;+ longs[longsIdx + 13] = l13;+ long l14 = (tmp[tmpIdx + 15] & MASK32_2) << 15;+ l14 |= (tmp[tmpIdx + 16] & MASK32_15) << 0;+ longs[longsIdx + 14] = l14;+ }+ }++ private static void decode18(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 36);+ shiftLongs(tmp, 36, longs, 0, 14, MASK32_18);+ for (int iter = 0, tmpIdx = 0, longsIdx = 36; iter < 4; ++iter, tmpIdx += 9, longsIdx += 7) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_14) << 4;+ l0 |= (tmp[tmpIdx + 1] >>> 10) & MASK32_4;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK32_10) << 8;+ l1 |= (tmp[tmpIdx + 2] >>> 6) & MASK32_8;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 2] & MASK32_6) << 12;+ l2 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_12;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 3] & MASK32_2) << 16;+ l3 |= (tmp[tmpIdx + 4] & MASK32_14) << 2;+ l3 |= (tmp[tmpIdx + 5] >>> 12) & MASK32_2;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 5] & MASK32_12) << 6;+ l4 |= (tmp[tmpIdx + 6] >>> 8) & MASK32_6;+ longs[longsIdx + 4] = l4;+ long l5 = (tmp[tmpIdx + 6] & MASK32_8) << 10;+ l5 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_10;+ longs[longsIdx + 5] = l5;+ long l6 = (tmp[tmpIdx + 7] & MASK32_4) << 14;+ l6 |= (tmp[tmpIdx + 8] & MASK32_14) << 0;+ longs[longsIdx + 6] = l6;+ }+ }++ private static void decode19(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 38);+ shiftLongs(tmp, 38, longs, 0, 13, MASK32_19);+ for (int iter = 0, tmpIdx = 0, longsIdx = 38; iter < 2; ++iter, tmpIdx += 19, longsIdx += 13) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_13) << 6;+ l0 |= (tmp[tmpIdx + 1] >>> 7) & MASK32_6;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK32_7) << 12;+ l1 |= (tmp[tmpIdx + 2] >>> 1) & MASK32_12;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 2] & MASK32_1) << 18;+ l2 |= (tmp[tmpIdx + 3] & MASK32_13) << 5;+ l2 |= (tmp[tmpIdx + 4] >>> 8) & MASK32_5;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 4] & MASK32_8) << 11;+ l3 |= (tmp[tmpIdx + 5] >>> 2) & MASK32_11;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 5] & MASK32_2) << 17;+ l4 |= (tmp[tmpIdx + 6] & MASK32_13) << 4;+ l4 |= (tmp[tmpIdx + 7] >>> 9) & MASK32_4;+ longs[longsIdx + 4] = l4;+ long l5 = (tmp[tmpIdx + 7] & MASK32_9) << 10;+ l5 |= (tmp[tmpIdx + 8] >>> 3) & MASK32_10;+ longs[longsIdx + 5] = l5;+ long l6 = (tmp[tmpIdx + 8] & MASK32_3) << 16;+ l6 |= (tmp[tmpIdx + 9] & MASK32_13) << 3;+ l6 |= (tmp[tmpIdx + 10] >>> 10) & MASK32_3;+ longs[longsIdx + 6] = l6;+ long l7 = (tmp[tmpIdx + 10] & MASK32_10) << 9;+ l7 |= (tmp[tmpIdx + 11] >>> 4) & MASK32_9;+ longs[longsIdx + 7] = l7;+ long l8 = (tmp[tmpIdx + 11] & MASK32_4) << 15;+ l8 |= (tmp[tmpIdx + 12] & MASK32_13) << 2;+ l8 |= (tmp[tmpIdx + 13] >>> 11) & MASK32_2;+ longs[longsIdx + 8] = l8;+ long l9 = (tmp[tmpIdx + 13] & MASK32_11) << 8;+ l9 |= (tmp[tmpIdx + 14] >>> 5) & MASK32_8;+ longs[longsIdx + 9] = l9;+ long l10 = (tmp[tmpIdx + 14] & MASK32_5) << 14;+ l10 |= (tmp[tmpIdx + 15] & MASK32_13) << 1;+ l10 |= (tmp[tmpIdx + 16] >>> 12) & MASK32_1;+ longs[longsIdx + 10] = l10;+ long l11 = (tmp[tmpIdx + 16] & MASK32_12) << 7;+ l11 |= (tmp[tmpIdx + 17] >>> 6) & MASK32_7;+ longs[longsIdx + 11] = l11;+ long l12 = (tmp[tmpIdx + 17] & MASK32_6) << 13;+ l12 |= (tmp[tmpIdx + 18] & MASK32_13) << 0;+ longs[longsIdx + 12] = l12;+ }+ }++ private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 40);+ shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);+ for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_12) << 8;+ l0 |= (tmp[tmpIdx + 1] >>> 4) & MASK32_8;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK32_4) << 16;+ l1 |= (tmp[tmpIdx + 2] & MASK32_12) << 4;+ l1 |= (tmp[tmpIdx + 3] >>> 8) & MASK32_4;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 3] & MASK32_8) << 12;+ l2 |= (tmp[tmpIdx + 4] & MASK32_12) << 0;+ longs[longsIdx + 2] = l2;+ }+ }++ private static void decode21(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 42);+ shiftLongs(tmp, 42, longs, 0, 11, MASK32_21);+ for (int iter = 0, tmpIdx = 0, longsIdx = 42; iter < 2; ++iter, tmpIdx += 21, longsIdx += 11) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_11) << 10;+ l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK32_10;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 1] & MASK32_1) << 20;+ l1 |= (tmp[tmpIdx + 2] & MASK32_11) << 9;+ l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_9;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 3] & MASK32_2) << 19;+ l2 |= (tmp[tmpIdx + 4] & MASK32_11) << 8;+ l2 |= (tmp[tmpIdx + 5] >>> 3) & MASK32_8;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 5] & MASK32_3) << 18;+ l3 |= (tmp[tmpIdx + 6] & MASK32_11) << 7;+ l3 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_7;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 7] & MASK32_4) << 17;+ l4 |= (tmp[tmpIdx + 8] & MASK32_11) << 6;+ l4 |= (tmp[tmpIdx + 9] >>> 5) & MASK32_6;+ longs[longsIdx + 4] = l4;+ long l5 = (tmp[tmpIdx + 9] & MASK32_5) << 16;+ l5 |= (tmp[tmpIdx + 10] & MASK32_11) << 5;+ l5 |= (tmp[tmpIdx + 11] >>> 6) & MASK32_5;+ longs[longsIdx + 5] = l5;+ long l6 = (tmp[tmpIdx + 11] & MASK32_6) << 15;+ l6 |= (tmp[tmpIdx + 12] & MASK32_11) << 4;+ l6 |= (tmp[tmpIdx + 13] >>> 7) & MASK32_4;+ longs[longsIdx + 6] = l6;+ long l7 = (tmp[tmpIdx + 13] & MASK32_7) << 14;+ l7 |= (tmp[tmpIdx + 14] & MASK32_11) << 3;+ l7 |= (tmp[tmpIdx + 15] >>> 8) & MASK32_3;+ longs[longsIdx + 7] = l7;+ long l8 = (tmp[tmpIdx + 15] & MASK32_8) << 13;+ l8 |= (tmp[tmpIdx + 16] & MASK32_11) << 2;+ l8 |= (tmp[tmpIdx + 17] >>> 9) & MASK32_2;+ longs[longsIdx + 8] = l8;+ long l9 = (tmp[tmpIdx + 17] & MASK32_9) << 12;+ l9 |= (tmp[tmpIdx + 18] & MASK32_11) << 1;+ l9 |= (tmp[tmpIdx + 19] >>> 10) & MASK32_1;+ longs[longsIdx + 9] = l9;+ long l10 = (tmp[tmpIdx + 19] & MASK32_10) << 11;+ l10 |= (tmp[tmpIdx + 20] & MASK32_11) << 0;+ longs[longsIdx + 10] = l10;+ }+ }++ private static void decode22(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 44);+ shiftLongs(tmp, 44, longs, 0, 10, MASK32_22);+ for (int iter = 0, tmpIdx = 0, longsIdx = 44; iter < 4; ++iter, tmpIdx += 11, longsIdx += 5) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_10) << 12;+ l0 |= (tmp[tmpIdx + 1] & MASK32_10) << 2;+ l0 |= (tmp[tmpIdx + 2] >>> 8) & MASK32_2;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 2] & MASK32_8) << 14;+ l1 |= (tmp[tmpIdx + 3] & MASK32_10) << 4;+ l1 |= (tmp[tmpIdx + 4] >>> 6) & MASK32_4;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 4] & MASK32_6) << 16;+ l2 |= (tmp[tmpIdx + 5] & MASK32_10) << 6;+ l2 |= (tmp[tmpIdx + 6] >>> 4) & MASK32_6;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 6] & MASK32_4) << 18;+ l3 |= (tmp[tmpIdx + 7] & MASK32_10) << 8;+ l3 |= (tmp[tmpIdx + 8] >>> 2) & MASK32_8;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 8] & MASK32_2) << 20;+ l4 |= (tmp[tmpIdx + 9] & MASK32_10) << 10;+ l4 |= (tmp[tmpIdx + 10] & MASK32_10) << 0;+ longs[longsIdx + 4] = l4;+ }+ }++ private static void decode23(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 46);+ shiftLongs(tmp, 46, longs, 0, 9, MASK32_23);+ for (int iter = 0, tmpIdx = 0, longsIdx = 46; iter < 2; ++iter, tmpIdx += 23, longsIdx += 9) {+ long l0 = (tmp[tmpIdx + 0] & MASK32_9) << 14;+ l0 |= (tmp[tmpIdx + 1] & MASK32_9) << 5;+ l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK32_5;+ longs[longsIdx + 0] = l0;+ long l1 = (tmp[tmpIdx + 2] & MASK32_4) << 19;+ l1 |= (tmp[tmpIdx + 3] & MASK32_9) << 10;+ l1 |= (tmp[tmpIdx + 4] & MASK32_9) << 1;+ l1 |= (tmp[tmpIdx + 5] >>> 8) & MASK32_1;+ longs[longsIdx + 1] = l1;+ long l2 = (tmp[tmpIdx + 5] & MASK32_8) << 15;+ l2 |= (tmp[tmpIdx + 6] & MASK32_9) << 6;+ l2 |= (tmp[tmpIdx + 7] >>> 3) & MASK32_6;+ longs[longsIdx + 2] = l2;+ long l3 = (tmp[tmpIdx + 7] & MASK32_3) << 20;+ l3 |= (tmp[tmpIdx + 8] & MASK32_9) << 11;+ l3 |= (tmp[tmpIdx + 9] & MASK32_9) << 2;+ l3 |= (tmp[tmpIdx + 10] >>> 7) & MASK32_2;+ longs[longsIdx + 3] = l3;+ long l4 = (tmp[tmpIdx + 10] & MASK32_7) << 16;+ l4 |= (tmp[tmpIdx + 11] & MASK32_9) << 7;+ l4 |= (tmp[tmpIdx + 12] >>> 2) & MASK32_7;+ longs[longsIdx + 4] = l4;+ long l5 = (tmp[tmpIdx + 12] & MASK32_2) << 21;+ l5 |= (tmp[tmpIdx + 13] & MASK32_9) << 12;+ l5 |= (tmp[tmpIdx + 14] & MASK32_9) << 3;+ l5 |= (tmp[tmpIdx + 15] >>> 6) & MASK32_3;+ longs[longsIdx + 5] = l5;+ long l6 = (tmp[tmpIdx + 15] & MASK32_6) << 17;+ l6 |= (tmp[tmpIdx + 16] & MASK32_9) << 8;+ l6 |= (tmp[tmpIdx + 17] >>> 1) & MASK32_8;+ longs[longsIdx + 6] = l6;+ long l7 = (tmp[tmpIdx + 17] & MASK32_1) << 22;+ l7 |= (tmp[tmpIdx + 18] & MASK32_9) << 13;+ l7 |= (tmp[tmpIdx + 19] & MASK32_9) << 4;+ l7 |= (tmp[tmpIdx + 20] >>> 5) & MASK32_4;+ longs[longsIdx + 7] = l7;+ long l8 = (tmp[tmpIdx + 20] & MASK32_5) << 18;+ l8 |= (tmp[tmpIdx + 21] & MASK32_9) << 9;+ l8 |= (tmp[tmpIdx + 22] & MASK32_9) << 0;+ longs[longsIdx + 8] = l8;+ }+ }++ private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {+ in.readLongs(tmp, 0, 48);+ shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);+ shiftLongs(tmp, 48, tmp, 0, 0, MASK32_8);+ for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {+ long l0 = tmp[tmpIdx + 0] << 16;+ l0 |= tmp[tmpIdx + 1] << 8;+ l0 |= tmp[tmpIdx + 2] << 0;+ longs[longsIdx + 0] = l0;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.javaindex 9633c386ec1..ce6c40f9402 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java@@ -98,7 +98,7 @@ public class Lucene99Codec extends Codec {super("Lucene99");this.storedFieldsFormat =new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);- this.defaultPostingsFormat = new Lucene90PostingsFormat();+ this.defaultPostingsFormat = new Lucene99PostingsFormat();this.defaultDVFormat = new Lucene90DocValuesFormat();this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();}@@ -151,7 +151,7 @@ public class Lucene99Codec extends Codec {/*** Returns the postings format that should be used for writing new segments of <code>field</code>.*- * <p>The default implementation always returns "Lucene90".+ * <p>The default implementation always returns "Lucene99".** <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:* future version of Lucene are only guaranteed to be able to read the default implementation,@@ -164,7 +164,7 @@ public class Lucene99Codec extends Codec {* Returns the docvalues format that should be used for writing new segments of <code>field</code>* .*- * <p>The default implementation always returns "Lucene90".+ * <p>The default implementation always returns "Lucene99".** <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:* future version of Lucene are only guaranteed to be able to read the default implementation.diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.javanew file mode 100644index 00000000000..f233276c6c5--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java@@ -0,0 +1,518 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.FieldsConsumer;+import org.apache.lucene.codecs.FieldsProducer;+import org.apache.lucene.codecs.MultiLevelSkipListWriter;+import org.apache.lucene.codecs.PostingsFormat;+import org.apache.lucene.codecs.PostingsReaderBase;+import org.apache.lucene.codecs.PostingsWriterBase;+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.index.TermState;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.packed.PackedInts;++/**+ * Lucene 9.9 postings format, which encodes postings in packed integer blocks for fast decode.+ *+ * <p>Basic idea:+ *+ * <ul>+ * <li><b>Packed Blocks and VInt Blocks</b>:+ * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed+ * format}): the block size (i.e. number of integers inside block) is fixed (currently 128).+ * Additionally blocks that are all the same value are encoded in an optimized way.+ * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block+ * size is variable.+ * <li><b>Block structure</b>:+ * <p>When the postings are long enough, Lucene99PostingsFormat will try to encode most+ * integer data as a packed block.+ * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as+ * two packed blocks, while the remaining 3 are encoded as one VInt block.+ * <p>Different kinds of data are always encoded separately into different packed blocks, but+ * may possibly be interleaved into the same VInt block.+ * <p>This strategy is applied to pairs: <document number, frequency>, <position,+ * payload length>, <position, offset start, offset length>, and <position,+ * payload length, offsetstart, offset length>.+ * <li><b>Skipdata settings</b>:+ * <p>The structure of skip table is quite similar to previous version of Lucene. Skip+ * interval is the same as block size, and each skip entry points to the beginning of each+ * block. However, for the first block, skip data is omitted.+ * <li><b>Positions, Payloads, and Offsets</b>:+ * <p>A position is an integer indicating where the term occurs within one document. A payload+ * is a blob of metadata associated with current position. An offset is a pair of integers+ * indicating the tokenized start/end offsets for given term in current position: it is+ * essentially a specialized payload.+ * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets+ * (assuming a null payload contributes one count). As mentioned in block structure, it is+ * possible to encode these three either combined or separately.+ * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,+ * position data is separated out as .pos, while payloads and offsets are encoded in .pay+ * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all+ * these three are stored interleaved into the .pos (so is payload metadata).+ * <p>With this strategy, the majority of payload and offset data will be outside .pos file.+ * So for queries that require only position data, running on a full index with payloads and+ * offsets, this reduces disk pre-fetches.+ * </ul>+ *+ * <p>Files and detailed format:+ *+ * <ul>+ * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>+ * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>+ * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>+ * <li><code>.pos</code>: <a href="#Positions">Positions</a>+ * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>+ * </ul>+ *+ * <a id="Termdictionary"></a>+ *+ * <dl>+ * <dd><b>Term Dictionary</b>+ * <p>The .tim file contains the list of terms in each field along with per-term statistics+ * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the+ * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on+ * the format.+ * <p>NOTE: The term dictionary can plug into different postings implementations: the postings+ * writer/reader are actually responsible for encoding and decoding the PostingsHeader and+ * TermMetadata sections described here:+ * <ul>+ * <li>PostingsHeader --> Header, PackedBlockSize+ * <li>TermMetadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,+ * PayFPDelta?, SkipFPDelta?+ * <li>Header, --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}+ * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> {@link+ * DataOutput#writeVLong VLong}+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version+ * information for the postings.+ * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width+ * is determined by the largest integer. Smaller block size result in smaller variance+ * among width of integers hence smaller indexes. Larger block size result in more+ * efficient bulk i/o hence better acceleration. This value should always be a multiple+ * of 64, currently fixed as 128 as a tradeoff. It is also the skip interval used to+ * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.+ * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In+ * particular, it is the difference of file offset between this term's data and previous+ * term's data (or zero, for the first term in the block).On disk it is stored as the+ * difference from previous value in sequence.+ * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.+ * While PayFPDelta determines the position of this term's <TermPayloads,+ * TermOffsets?> within the .pay file. Similar to DocFPDelta, it is the difference+ * between two file positions (or neglected, for fields that omit payloads and offsets).+ * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last+ * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or+ * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to+ * load following payloads and offsets from .pos instead of .pay. Every time a new block+ * of positions are to be loaded, the PostingsReader will use this value to check+ * whether current block is packed format or VInt. When packed format, payloads and+ * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when+ * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).+ * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In+ * particular, it is the length of the TermFreq data. SkipDelta is only stored if+ * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene99PostingsFormat).+ * <li>SingletonDocID is an optimization when a term only appears in one document. In this+ * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a+ * VIntBlock at that location, the single document ID is written to the term dictionary.+ * </ul>+ * </dl>+ *+ * <a id="Termindex"></a>+ *+ * <dl>+ * <dd><b>Term Index</b>+ * <p>The .tip file contains an index into the term dictionary, so that it can be accessed+ * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.+ * </dl>+ *+ * <a id="Frequencies"></a>+ *+ * <dl>+ * <dd><b>Frequencies and Skip Data</b>+ * <p>The .doc file contains the lists of documents which contain each term, along with the+ * frequency of the term in that document (except when frequencies are omitted: {@link+ * IndexOptions#DOCS}). It also saves skip data to the beginning of each packed or VInt block,+ * when the length of document list is larger than packed block size.+ * <ul>+ * <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>TermFreqs --> <PackedBlock> <sup>PackedDocBlockNum</sup>, VIntBlock?+ * <li>PackedBlock --> PackedDocDeltaBlock, PackedFreqBlock?+ * <li>VIntBlock --> <DocDelta[,+ * Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>+ * <li>SkipData --> <<SkipLevelLength, SkipLevel> <sup>NumSkipLevels-1</sup>,+ * SkipLevel>, SkipDatum?+ * <li>SkipLevel --> <SkipDatum> <sup>TrimmedDocFreq/(PackedBlockSize^(Level ++ * 1))</sup>+ * <li>SkipDatum --> DocSkip, DocFPSkip, <PosFPSkip, PosBlockOffset, PayLength?,+ * PayFPSkip?>?, ImpactLength, <CompetitiveFreqDelta, CompetitiveNormDelta?>+ * <sup>ImpactCount</sup>, SkipChildLevelPointer?+ * <li>PackedDocDeltaBlock, PackedFreqBlock --> {@link PackedInts PackedInts}+ * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto,+ * PayFPSkip, ImpactLength, CompetitiveFreqDelta --> {@link DataOutput#writeVInt+ * VInt}+ * <li>CompetitiveNormDelta --> {@link DataOutput#writeZLong ZLong}+ * <li>SkipChildLevelPointer --> {@link DataOutput#writeVLong VLong}+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>PackedDocDeltaBlock is theoretically generated from two steps:+ * <ol>+ * <li>Calculate the difference between each document number and previous one, and get+ * a d-gaps list (for the first document, use absolute value);+ * <li>For those d-gaps from first one to+ * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed+ * blocks.+ * </ol>+ * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.+ * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a+ * format that encodes DocDelta and Freq:+ * <p>DocDelta: if frequencies are indexed, this determines both the document number and+ * the frequency. In particular, DocDelta/2 is the difference between this document+ * number and the previous document number (or zero when this is the first document in a+ * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the+ * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the+ * gap (not multiplied by 2) between document numbers and no frequency information is+ * stored.+ * <p>For example, the TermFreqs for a term which occurs once in document seven and+ * three times in document eleven, with frequencies indexed, would be the following+ * sequence of VInts:+ * <p>15, 8, 3+ * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence+ * of VInts instead:+ * <p>7,4+ * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or+ * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)+ * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this+ * trick since the definition of skip entry is a little different from base interface.+ * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for+ * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.+ * However, in Lucene99PostingsFormat, the skip data is saved for+ * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting+ * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of+ * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than+ * Lucene99SkipWriter.+ * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or+ * VInt), it is omitted.+ * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document+ * number in the postings (i.e. last document number in each packed block). On disk it+ * is stored as the difference from previous value in the sequence.+ * <li>DocFPSkip records the file offsets of each block (excluding )posting at+ * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.+ * The file offsets are relative to the start of current term's TermFreqs. On disk it is+ * also stored as the difference from previous SkipDatum in the sequence.+ * <li>Since positions and payloads are also block encoded, the skip should skip to related+ * block first, then fetch the values according to in-block offset. PosFPSkip and+ * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.+ * While PosBlockOffset indicates which value to fetch inside the related block+ * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as+ * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,+ * and stored as a difference sequence.+ * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to+ * the sum of the payload lengths in the current block up to PosBlockOffset+ * <li>ImpactLength is the total length of CompetitiveFreqDelta and CompetitiveNormDelta+ * pairs. CompetitiveFreqDelta and CompetitiveNormDelta are used to safely skip score+ * calculation for uncompetitive documents; See {@link+ * org.apache.lucene.codecs.CompetitiveImpactAccumulator} for more details.+ * </ul>+ * </dl>+ *+ * <a id="Positions"></a>+ *+ * <dl>+ * <dd><b>Positions</b>+ * <p>The .pos file contains the lists of positions that each term occurs at within documents.+ * It also sometimes stores part of payloads and offsets for speedup.+ * <ul>+ * <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>TermPositions --> <PackedPosDeltaBlock> <sup>PackedPosBlockNum</sup>,+ * VIntBlock?+ * <li>VIntBlock --> <PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,+ * OffsetLength?><sup>PosVIntCount</sup>+ * <li>PackedPosDeltaBlock --> {@link PackedInts PackedInts}+ * <li>PositionDelta, OffsetDelta, OffsetLength --> {@link DataOutput#writeVInt VInt}+ * <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayLength</sup>+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and+ * position values for each term document pair are incremental, and ordered by document+ * number.+ * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,+ * payloads or offsets. In particular, PackedPosBlockNum =+ * floor(totalTermFreq/PackedBlockSize)+ * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,+ * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize+ * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock+ * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.+ * <li>PositionDelta is, if payloads are disabled for the term's field, the difference+ * between the position of the current occurrence in the document and the previous+ * occurrence (or zero, if this is the first occurrence in this document). If payloads+ * are enabled for the term's field, then PositionDelta/2 is the difference between the+ * current and the previous position. If payloads are enabled and PositionDelta is odd,+ * then PayloadLength is stored, indicating the length of the payload at the current+ * term position.+ * <li>For example, the TermPositions for a term which occurs as the fourth term in one+ * document, and as the fifth and ninth term in a subsequent document, would be the+ * following sequence of VInts (payloads disabled):+ * <p>4, 5, 4+ * <li>PayloadData is metadata associated with the current term position. If PayloadLength+ * is stored at the current position, then it indicates the length of this payload. If+ * PayloadLength is not stored, then this payload has the same length as the payload at+ * the previous position.+ * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous+ * occurrence (or zero, if this is the first occurrence in this document). If+ * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous+ * occurrence and an OffsetLength follows. Offset data is only written for {@link+ * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.+ * </ul>+ * </dl>+ *+ * <a id="Payloads"></a>+ *+ * <dl>+ * <dd><b>Payloads and Offsets</b>+ * <p>The .pay file will store payloads and offsets associated with certain term-document+ * positions. Some payloads and offsets will be separated out into .pos file, for performance+ * reasons.+ * <ul>+ * <li>PayFile(.pay): --> Header, <TermPayloads?, TermOffsets?>+ * <sup>TermCount</sup>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>TermPayloads --> <PackedPayLengthBlock, SumPayLength, PayData>+ * <sup>PackedPayBlockNum</sup>+ * <li>TermOffsets --> <PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock>+ * <sup>PackedPayBlockNum</sup>+ * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock -->+ * {@link PackedInts PackedInts}+ * <li>SumPayLength --> {@link DataOutput#writeVInt VInt}+ * <li>PayData --> {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <p>Notes:+ * <ul>+ * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that+ * part of payload/offsets are stored in .pos.+ * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is+ * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip+ * Data</a>. While PackedStartDeltaBlock follows a same procedure as+ * PackedDocDeltaBlock.+ * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also+ * synonym for PackedOffsetBlockNum.+ * <li>SumPayLength is the total length of payloads written within one block, should be the+ * sum of PayLengths in one packed block.+ * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the+ * current position.+ * </ul>+ * </dl>+ *+ * @lucene.experimental+ */+public final class Lucene99PostingsFormat extends PostingsFormat {++ /**+ * Filename extension for document number, frequencies, and skip data. See chapter: <a+ * href="#Frequencies">Frequencies and Skip Data</a>+ */+ public static final String DOC_EXTENSION = "doc";++ /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */+ public static final String POS_EXTENSION = "pos";++ /**+ * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and+ * Offsets</a>+ */+ public static final String PAY_EXTENSION = "pay";++ /** Size of blocks. */+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;++ /**+ * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,+ * but slower skipping in big posting lists.+ */+ static final int MAX_SKIP_LEVELS = 10;++ static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";+ static final String DOC_CODEC = "Lucene99PostingsWriterDoc";+ static final String POS_CODEC = "Lucene99PostingsWriterPos";+ static final String PAY_CODEC = "Lucene99PostingsWriterPay";++ // Increment version to change it+ static final int VERSION_START = 0;+ static final int VERSION_CURRENT = VERSION_START;++ private final int minTermBlockSize;+ private final int maxTermBlockSize;++ /** Creates {@code Lucene99PostingsFormat} with default settings. */+ public Lucene99PostingsFormat() {+ this(+ Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,+ Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);+ }++ /**+ * Creates {@code Lucene99PostingsFormat} with custom values for {@code minBlockSize} and {@code+ * maxBlockSize} passed to block terms dictionary.+ *+ * @see+ * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)+ */+ public Lucene99PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {+ super("Lucene99");+ Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);+ this.minTermBlockSize = minTermBlockSize;+ this.maxTermBlockSize = maxTermBlockSize;+ }++ @Override+ public String toString() {+ return getName();+ }++ @Override+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);+ boolean success = false;+ try {+ FieldsConsumer ret =+ new Lucene90BlockTreeTermsWriter(+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);+ success = true;+ return ret;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(postingsWriter);+ }+ }+ }++ @Override+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);+ boolean success = false;+ try {+ FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);+ success = true;+ return ret;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(postingsReader);+ }+ }+ }++ /**+ * Holds all state required for {@link Lucene99PostingsReader} to produce a {@link+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.+ *+ * @lucene.internal+ */+ public static final class IntBlockTermState extends BlockTermState {+ /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */+ public long docStartFP;++ /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */+ public long posStartFP;++ /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */+ public long payStartFP;++ /**+ * file offset for the start of the skip list, relative to docStartFP, if there are more than+ * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1+ */+ public long skipOffset;++ /**+ * file offset for the last position in the last block, if there are more than {@link+ * ForUtil#BLOCK_SIZE} positions; otherwise -1+ *+ * <p>One might think to use total term frequency to track how many positions are left to read+ * as we decode the blocks, and decode the last block differently when num_left_positions <+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip+ * blocks as the skipper will only tell us new position offset (start of block) and number of+ * positions to skip for that block, without telling us how many positions it has skipped.+ */+ public long lastPosBlockOffset;++ /**+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly+ * totalTermFreq in this case.+ */+ public int singletonDocID;++ /** Sole constructor. */+ public IntBlockTermState() {+ skipOffset = -1;+ lastPosBlockOffset = -1;+ singletonDocID = -1;+ }++ @Override+ public IntBlockTermState clone() {+ IntBlockTermState other = new IntBlockTermState();+ other.copyFrom(this);+ return other;+ }++ @Override+ public void copyFrom(TermState _other) {+ super.copyFrom(_other);+ IntBlockTermState other = (IntBlockTermState) _other;+ docStartFP = other.docStartFP;+ posStartFP = other.posStartFP;+ payStartFP = other.payStartFP;+ lastPosBlockOffset = other.lastPosBlockOffset;+ skipOffset = other.skipOffset;+ singletonDocID = other.singletonDocID;+ }++ @Override+ public String toString() {+ return super.toString()+ + " docStartFP="+ + docStartFP+ + " posStartFP="+ + posStartFP+ + " payStartFP="+ + payStartFP+ + " lastPosBlockOffset="+ + lastPosBlockOffset+ + " singletonDocID="+ + singletonDocID;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.javanew file mode 100644index 00000000000..77e50e23ce8--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java@@ -0,0 +1,2087 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_START;++import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.PostingsReaderBase;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.Impacts;+import org.apache.lucene.index.ImpactsEnum;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.SlowImpactsEnum;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BitUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;++/**+ * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.+ *+ * @lucene.experimental+ */+public final class Lucene99PostingsReader extends PostingsReaderBase {++ private final IndexInput docIn;+ private final IndexInput posIn;+ private final IndexInput payIn;++ private final int version;++ /** Sole constructor. */+ public Lucene99PostingsReader(SegmentReadState state) throws IOException {+ boolean success = false;+ IndexInput docIn = null;+ IndexInput posIn = null;+ IndexInput payIn = null;++ // NOTE: these data files are too costly to verify checksum against all the bytes on open,+ // but for now we at least verify proper structure of the checksum footer: which looks+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption+ // such as file truncation.++ String docName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.DOC_EXTENSION);+ try {+ docIn = state.directory.openInput(docName, state.context);+ version =+ CodecUtil.checkIndexHeader(+ docIn,+ DOC_CODEC,+ VERSION_START,+ VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ CodecUtil.retrieveChecksum(docIn);++ if (state.fieldInfos.hasProx()) {+ String proxName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.POS_EXTENSION);+ posIn = state.directory.openInput(proxName, state.context);+ CodecUtil.checkIndexHeader(+ posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);+ CodecUtil.retrieveChecksum(posIn);++ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {+ String payName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name,+ state.segmentSuffix,+ Lucene99PostingsFormat.PAY_EXTENSION);+ payIn = state.directory.openInput(payName, state.context);+ CodecUtil.checkIndexHeader(+ payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);+ CodecUtil.retrieveChecksum(payIn);+ }+ }++ this.docIn = docIn;+ this.posIn = posIn;+ this.payIn = payIn;+ success = true;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(docIn, posIn, payIn);+ }+ }+ }++ @Override+ public void init(IndexInput termsIn, SegmentReadState state) throws IOException {+ // Make sure we are talking to the matching postings writer+ CodecUtil.checkIndexHeader(+ termsIn,+ TERMS_CODEC,+ VERSION_START,+ VERSION_CURRENT,+ state.segmentInfo.getId(),+ state.segmentSuffix);+ final int indexBlockSize = termsIn.readVInt();+ if (indexBlockSize != BLOCK_SIZE) {+ throw new IllegalStateException(+ "index-time BLOCK_SIZE ("+ + indexBlockSize+ + ") != read-time BLOCK_SIZE ("+ + BLOCK_SIZE+ + ")");+ }+ }++ /** Read values that have been written using variable-length encoding instead of bit-packing. */+ static void readVIntBlock(+ IndexInput docIn, long[] docBuffer, long[] freqBuffer, int num, boolean indexHasFreq)+ throws IOException {+ if (indexHasFreq) {+ for (int i = 0; i < num; i++) {+ final int code = docIn.readVInt();+ docBuffer[i] = code >>> 1;+ if ((code & 1) != 0) {+ freqBuffer[i] = 1;+ } else {+ freqBuffer[i] = docIn.readVInt();+ }+ }+ } else {+ for (int i = 0; i < num; i++) {+ docBuffer[i] = docIn.readVInt();+ }+ }+ }++ static void prefixSum(long[] buffer, int count, long base) {+ buffer[0] += base;+ for (int i = 1; i < count; ++i) {+ buffer[i] += buffer[i - 1];+ }+ }++ static int findFirstGreater(long[] buffer, int target, int from) {+ for (int i = from; i < BLOCK_SIZE; ++i) {+ if (buffer[i] >= target) {+ return i;+ }+ }+ return BLOCK_SIZE;+ }++ @Override+ public BlockTermState newTermState() {+ return new IntBlockTermState();+ }++ @Override+ public void close() throws IOException {+ IOUtils.close(docIn, posIn, payIn);+ }++ @Override+ public void decodeTerm(+ DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)+ throws IOException {+ final IntBlockTermState termState = (IntBlockTermState) _termState;+ final boolean fieldHasPositions =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;+ final boolean fieldHasOffsets =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ final boolean fieldHasPayloads = fieldInfo.hasPayloads();++ if (absolute) {+ termState.docStartFP = 0;+ termState.posStartFP = 0;+ termState.payStartFP = 0;+ }++ final long l = in.readVLong();+ if ((l & 0x01) == 0) {+ termState.docStartFP += l >>> 1;+ if (termState.docFreq == 1) {+ termState.singletonDocID = in.readVInt();+ } else {+ termState.singletonDocID = -1;+ }+ } else {+ assert absolute == false;+ assert termState.singletonDocID != -1;+ termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);+ }++ if (fieldHasPositions) {+ termState.posStartFP += in.readVLong();+ if (fieldHasOffsets || fieldHasPayloads) {+ termState.payStartFP += in.readVLong();+ }+ if (termState.totalTermFreq > BLOCK_SIZE) {+ termState.lastPosBlockOffset = in.readVLong();+ } else {+ termState.lastPosBlockOffset = -1;+ }+ }++ if (termState.docFreq > BLOCK_SIZE) {+ termState.skipOffset = in.readVLong();+ } else {+ termState.skipOffset = -1;+ }+ }++ @Override+ public PostingsEnum postings(+ FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)+ throws IOException {++ boolean indexHasPositions =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;++ if (indexHasPositions == false+ || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {+ BlockDocsEnum docsEnum;+ if (reuse instanceof BlockDocsEnum) {+ docsEnum = (BlockDocsEnum) reuse;+ if (!docsEnum.canReuse(docIn, fieldInfo)) {+ docsEnum = new BlockDocsEnum(fieldInfo);+ }+ } else {+ docsEnum = new BlockDocsEnum(fieldInfo);+ }+ return docsEnum.reset((IntBlockTermState) termState, flags);+ } else {+ EverythingEnum everythingEnum;+ if (reuse instanceof EverythingEnum) {+ everythingEnum = (EverythingEnum) reuse;+ if (!everythingEnum.canReuse(docIn, fieldInfo)) {+ everythingEnum = new EverythingEnum(fieldInfo);+ }+ } else {+ everythingEnum = new EverythingEnum(fieldInfo);+ }+ return everythingEnum.reset((IntBlockTermState) termState, flags);+ }+ }++ @Override+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)+ throws IOException {+ if (state.docFreq <= BLOCK_SIZE) {+ // no skip data+ return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));+ }++ final boolean indexHasPositions =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;+ final boolean indexHasOffsets =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ final boolean indexHasPayloads = fieldInfo.hasPayloads();++ if (indexHasPositions == false+ || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {+ return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state);+ }++ if (indexHasPositions+ && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)+ && (indexHasOffsets == false+ || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)+ && (indexHasPayloads == false+ || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {+ return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);+ }++ return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);+ }++ final class BlockDocsEnum extends PostingsEnum {++ final ForUtil forUtil = new ForUtil();+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);+ final PForUtil pforUtil = new PForUtil(forUtil);++ private final long[] docBuffer = new long[BLOCK_SIZE + 1];+ private final long[] freqBuffer = new long[BLOCK_SIZE];++ private int docBufferUpto;++ private Lucene99SkipReader skipper;+ private boolean skipped;++ final IndexInput startDocIn;++ IndexInput docIn;+ final boolean indexHasFreq;+ final boolean indexHasPos;+ final boolean indexHasOffsets;+ final boolean indexHasPayloads;++ private int docFreq; // number of docs in this posting list+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)+ private int blockUpto; // number of docs in or before the current block+ private int doc; // doc we last read+ private long accum; // accumulator for doc deltas++ // Where this term's postings start in the .doc file:+ private long docTermStartFP;++ // Where this term's skip data starts (after+ // docTermStartFP) in the .doc file (or -1 if there is+ // no skip data for this term):+ private long skipOffset;++ // docID for next skip point, we won't use skipper if+ // target docID is not larger than this+ private int nextSkipDoc;++ private boolean needsFreq; // true if the caller actually needs frequencies+ // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block+ // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer+ // (needsFreq=false)+ private boolean isFreqsRead;+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1++ public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {+ this.startDocIn = Lucene99PostingsReader.this.docIn;+ this.docIn = null;+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;+ indexHasPos =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;+ indexHasOffsets =+ fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ indexHasPayloads = fieldInfo.hasPayloads();+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in+ // advance()+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;+ }++ public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {+ return docIn == startDocIn+ && indexHasFreq+ == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)+ && indexHasPos+ == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)+ >= 0)+ && indexHasPayloads == fieldInfo.hasPayloads();+ }++ public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {+ docFreq = termState.docFreq;+ totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;+ docTermStartFP = termState.docStartFP;+ skipOffset = termState.skipOffset;+ singletonDocID = termState.singletonDocID;+ if (docFreq > 1) {+ if (docIn == null) {+ // lazy init+ docIn = startDocIn.clone();+ }+ docIn.seek(docTermStartFP);+ }++ doc = -1;+ this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);+ this.isFreqsRead = true;+ if (indexHasFreq == false || needsFreq == false) {+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ freqBuffer[i] = 1;+ }+ }+ accum = 0;+ blockUpto = 0;+ nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block+ docBufferUpto = BLOCK_SIZE;+ skipped = false;+ return this;+ }++ @Override+ public int freq() throws IOException {+ if (isFreqsRead == false) {+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block+ isFreqsRead = true;+ }+ return (int) freqBuffer[docBufferUpto - 1];+ }++ @Override+ public int nextPosition() throws IOException {+ return -1;+ }++ @Override+ public int startOffset() throws IOException {+ return -1;+ }++ @Override+ public int endOffset() throws IOException {+ return -1;+ }++ @Override+ public BytesRef getPayload() throws IOException {+ return null;+ }++ @Override+ public int docID() {+ return doc;+ }++ private void refillDocs() throws IOException {+ // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn+ // after it+ if (isFreqsRead == false) {+ pforUtil.skip(docIn);+ isFreqsRead = true;+ }++ final int left = docFreq - blockUpto;+ assert left >= 0;++ if (left >= BLOCK_SIZE) {+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);++ if (indexHasFreq) {+ if (needsFreq) {+ isFreqsRead = false;+ } else {+ pforUtil.skip(docIn); // skip over freqBuffer if we don't need them at all+ }+ }+ blockUpto += BLOCK_SIZE;+ } else if (docFreq == 1) {+ docBuffer[0] = singletonDocID;+ freqBuffer[0] = totalTermFreq;+ docBuffer[1] = NO_MORE_DOCS;+ blockUpto++;+ } else {+ // Read vInts:+ readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);+ prefixSum(docBuffer, left, accum);+ docBuffer[left] = NO_MORE_DOCS;+ blockUpto += left;+ }+ accum = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;+ }++ @Override+ public int nextDoc() throws IOException {+ if (docBufferUpto == BLOCK_SIZE) {+ refillDocs(); // we don't need to load freqBuffer for now (will be loaded later if+ // necessary)+ }++ doc = (int) docBuffer[docBufferUpto];+ docBufferUpto++;+ return doc;+ }++ @Override+ public int advance(int target) throws IOException {+ // current skip docID < docIDs generated from current buffer <= next skip docID+ // we don't need to skip if target is buffered already+ if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {++ if (skipper == null) {+ // Lazy init: first time this enum has ever been used for skipping+ skipper =+ new Lucene99SkipReader(+ docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);+ }++ if (!skipped) {+ assert skipOffset != -1;+ // This is the first time this enum has skipped+ // since reset() was called; load the skip data:+ skipper.init(docTermStartFP + skipOffset, docTermStartFP, 0, 0, docFreq);+ skipped = true;+ }++ // always plus one to fix the result, since skip position in Lucene99SkipReader+ // is a little different from MultiLevelSkipListReader+ final int newDocUpto = skipper.skipTo(target) + 1;++ if (newDocUpto >= blockUpto) {+ // Skipper moved+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;+ blockUpto = newDocUpto;++ // Force to read next block+ docBufferUpto = BLOCK_SIZE;+ accum = skipper.getDoc(); // actually, this is just lastSkipEntry+ docIn.seek(skipper.getDocPointer()); // now point to the block we want to search+ // even if freqBuffer were not read from the previous block, we will mark them as read,+ // as we don't need to skip the previous block freqBuffer in refillDocs,+ // as we have already positioned docIn where in needs to be.+ isFreqsRead = true;+ }+ // next time we call advance, this is used to+ // foresee whether skipper is necessary.+ nextSkipDoc = skipper.getNextSkipDoc();+ }+ if (docBufferUpto == BLOCK_SIZE) {+ refillDocs();+ }++ // Now scan... this is an inlined/pared down version+ // of nextDoc():+ long doc;+ while (true) {+ doc = docBuffer[docBufferUpto];++ if (doc >= target) {+ break;+ }+ ++docBufferUpto;+ }++ docBufferUpto++;+ return this.doc = (int) doc;+ }++ @Override+ public long cost() {+ return docFreq;+ }+ }++ // Also handles payloads + offsets+ final class EverythingEnum extends PostingsEnum {++ final ForUtil forUtil = new ForUtil();+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);+ final PForUtil pforUtil = new PForUtil(forUtil);++ private final long[] docBuffer = new long[BLOCK_SIZE + 1];+ private final long[] freqBuffer = new long[BLOCK_SIZE + 1];+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];++ private final long[] payloadLengthBuffer;+ private final long[] offsetStartDeltaBuffer;+ private final long[] offsetLengthBuffer;++ private byte[] payloadBytes;+ private int payloadByteUpto;+ private int payloadLength;++ private int lastStartOffset;+ private int startOffset;+ private int endOffset;++ private int docBufferUpto;+ private int posBufferUpto;++ private Lucene99SkipReader skipper;+ private boolean skipped;++ final IndexInput startDocIn;++ IndexInput docIn;+ final IndexInput posIn;+ final IndexInput payIn;+ final BytesRef payload;++ final boolean indexHasOffsets;+ final boolean indexHasPayloads;++ private int docFreq; // number of docs in this posting list+ private long totalTermFreq; // number of positions in this posting list+ private int blockUpto; // number of docs in or before the current block+ private int doc; // doc we last read+ private long accum; // accumulator for doc deltas+ private int freq; // freq we last read+ private int position; // current position++ // how many positions "behind" we are; nextPosition must+ // skip these to "catch up":+ private int posPendingCount;++ // Lazy pos seek: if != -1 then we must seek to this FP+ // before reading positions:+ private long posPendingFP;++ // Lazy pay seek: if != -1 then we must seek to this FP+ // before reading payloads/offsets:+ private long payPendingFP;++ // Where this term's postings start in the .doc file:+ private long docTermStartFP;++ // Where this term's postings start in the .pos file:+ private long posTermStartFP;++ // Where this term's payloads/offsets start in the .pay+ // file:+ private long payTermStartFP;++ // File pointer where the last (vInt encoded) pos delta+ // block is. We need this to know whether to bulk+ // decode vs vInt decode the block:+ private long lastPosBlockFP;++ // Where this term's skip data starts (after+ // docTermStartFP) in the .doc file (or -1 if there is+ // no skip data for this term):+ private long skipOffset;++ private int nextSkipDoc;++ private boolean needsOffsets; // true if we actually need offsets+ private boolean needsPayloads; // true if we actually need payloads+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1++ public EverythingEnum(FieldInfo fieldInfo) throws IOException {+ indexHasOffsets =+ fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ indexHasPayloads = fieldInfo.hasPayloads();++ this.startDocIn = Lucene99PostingsReader.this.docIn;+ this.docIn = null;+ this.posIn = Lucene99PostingsReader.this.posIn.clone();+ if (indexHasOffsets || indexHasPayloads) {+ this.payIn = Lucene99PostingsReader.this.payIn.clone();+ } else {+ this.payIn = null;+ }+ if (indexHasOffsets) {+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];+ offsetLengthBuffer = new long[BLOCK_SIZE];+ } else {+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ startOffset = -1;+ endOffset = -1;+ }++ if (indexHasPayloads) {+ payloadLengthBuffer = new long[BLOCK_SIZE];+ payloadBytes = new byte[128];+ payload = new BytesRef();+ } else {+ payloadLengthBuffer = null;+ payloadBytes = null;+ payload = null;+ }++ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in+ // advance()+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;+ }++ public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {+ return docIn == startDocIn+ && indexHasOffsets+ == (fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0)+ && indexHasPayloads == fieldInfo.hasPayloads();+ }++ public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException {+ docFreq = termState.docFreq;+ docTermStartFP = termState.docStartFP;+ posTermStartFP = termState.posStartFP;+ payTermStartFP = termState.payStartFP;+ skipOffset = termState.skipOffset;+ totalTermFreq = termState.totalTermFreq;+ singletonDocID = termState.singletonDocID;+ if (docFreq > 1) {+ if (docIn == null) {+ // lazy init+ docIn = startDocIn.clone();+ }+ docIn.seek(docTermStartFP);+ }+ posPendingFP = posTermStartFP;+ payPendingFP = payTermStartFP;+ posPendingCount = 0;+ if (termState.totalTermFreq < BLOCK_SIZE) {+ lastPosBlockFP = posTermStartFP;+ } else if (termState.totalTermFreq == BLOCK_SIZE) {+ lastPosBlockFP = -1;+ } else {+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;+ }++ this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);+ this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);++ doc = -1;+ accum = 0;+ blockUpto = 0;+ if (docFreq > BLOCK_SIZE) {+ nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block+ } else {+ nextSkipDoc = NO_MORE_DOCS; // not enough docs for skipping+ }+ docBufferUpto = BLOCK_SIZE;+ skipped = false;+ return this;+ }++ @Override+ public int freq() throws IOException {+ return freq;+ }++ @Override+ public int docID() {+ return doc;+ }++ private void refillDocs() throws IOException {+ final int left = docFreq - blockUpto;+ assert left >= 0;++ if (left >= BLOCK_SIZE) {+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);+ pforUtil.decode(docIn, freqBuffer);+ blockUpto += BLOCK_SIZE;+ } else if (docFreq == 1) {+ docBuffer[0] = singletonDocID;+ freqBuffer[0] = totalTermFreq;+ docBuffer[1] = NO_MORE_DOCS;+ blockUpto++;+ } else {+ readVIntBlock(docIn, docBuffer, freqBuffer, left, true);+ prefixSum(docBuffer, left, accum);+ docBuffer[left] = NO_MORE_DOCS;+ blockUpto += left;+ }+ accum = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;+ }++ private void refillPositions() throws IOException {+ if (posIn.getFilePointer() == lastPosBlockFP) {+ final int count = (int) (totalTermFreq % BLOCK_SIZE);+ int payloadLength = 0;+ int offsetLength = 0;+ payloadByteUpto = 0;+ for (int i = 0; i < count; i++) {+ int code = posIn.readVInt();+ if (indexHasPayloads) {+ if ((code & 1) != 0) {+ payloadLength = posIn.readVInt();+ }+ payloadLengthBuffer[i] = payloadLength;+ posDeltaBuffer[i] = code >>> 1;+ if (payloadLength != 0) {+ if (payloadByteUpto + payloadLength > payloadBytes.length) {+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);+ }+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);+ payloadByteUpto += payloadLength;+ }+ } else {+ posDeltaBuffer[i] = code;+ }++ if (indexHasOffsets) {+ int deltaCode = posIn.readVInt();+ if ((deltaCode & 1) != 0) {+ offsetLength = posIn.readVInt();+ }+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;+ offsetLengthBuffer[i] = offsetLength;+ }+ }+ payloadByteUpto = 0;+ } else {+ pforUtil.decode(posIn, posDeltaBuffer);++ if (indexHasPayloads) {+ if (needsPayloads) {+ pforUtil.decode(payIn, payloadLengthBuffer);+ int numBytes = payIn.readVInt();++ if (numBytes > payloadBytes.length) {+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);+ }+ payIn.readBytes(payloadBytes, 0, numBytes);+ } else {+ // this works, because when writing a vint block we always force the first length to be+ // written+ pforUtil.skip(payIn); // skip over lengths+ int numBytes = payIn.readVInt(); // read length of payloadBytes+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes+ }+ payloadByteUpto = 0;+ }++ if (indexHasOffsets) {+ if (needsOffsets) {+ pforUtil.decode(payIn, offsetStartDeltaBuffer);+ pforUtil.decode(payIn, offsetLengthBuffer);+ } else {+ // this works, because when writing a vint block we always force the first length to be+ // written+ pforUtil.skip(payIn); // skip over starts+ pforUtil.skip(payIn); // skip over lengths+ }+ }+ }+ }++ @Override+ public int nextDoc() throws IOException {+ if (docBufferUpto == BLOCK_SIZE) {+ refillDocs();+ }++ doc = (int) docBuffer[docBufferUpto];+ freq = (int) freqBuffer[docBufferUpto];+ posPendingCount += freq;+ docBufferUpto++;++ position = 0;+ lastStartOffset = 0;+ return doc;+ }++ @Override+ public int advance(int target) throws IOException {+ if (target > nextSkipDoc) {+ if (skipper == null) {+ // Lazy init: first time this enum has ever been used for skipping+ skipper =+ new Lucene99SkipReader(+ docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);+ }++ if (!skipped) {+ assert skipOffset != -1;+ // This is the first time this enum has skipped+ // since reset() was called; load the skip data:+ skipper.init(+ docTermStartFP + skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);+ skipped = true;+ }++ final int newDocUpto = skipper.skipTo(target) + 1;++ if (newDocUpto > blockUpto - BLOCK_SIZE + docBufferUpto) {+ // Skipper moved+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;+ blockUpto = newDocUpto;++ // Force to read next block+ docBufferUpto = BLOCK_SIZE;+ accum = skipper.getDoc();+ docIn.seek(skipper.getDocPointer());+ posPendingFP = skipper.getPosPointer();+ payPendingFP = skipper.getPayPointer();+ posPendingCount = skipper.getPosBufferUpto();+ lastStartOffset = 0; // new document+ payloadByteUpto = skipper.getPayloadByteUpto();+ }+ nextSkipDoc = skipper.getNextSkipDoc();+ }+ if (docBufferUpto == BLOCK_SIZE) {+ refillDocs();+ }++ // Now scan:+ long doc;+ while (true) {+ doc = docBuffer[docBufferUpto];+ freq = (int) freqBuffer[docBufferUpto];+ posPendingCount += freq;+ docBufferUpto++;++ if (doc >= target) {+ break;+ }+ }++ position = 0;+ lastStartOffset = 0;+ return this.doc = (int) doc;+ }++ // TODO: in theory we could avoid loading frq block+ // when not needed, ie, use skip data to load how far to+ // seek the pos pointer ... instead of having to load frq+ // blocks only to sum up how many positions to skip+ private void skipPositions() throws IOException {+ // Skip positions now:+ int toSkip = posPendingCount - freq;+ // if (DEBUG) {+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);+ // }++ final int leftInBlock = BLOCK_SIZE - posBufferUpto;+ if (toSkip < leftInBlock) {+ int end = posBufferUpto + toSkip;+ while (posBufferUpto < end) {+ if (indexHasPayloads) {+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];+ }+ posBufferUpto++;+ }+ } else {+ toSkip -= leftInBlock;+ while (toSkip >= BLOCK_SIZE) {+ assert posIn.getFilePointer() != lastPosBlockFP;+ pforUtil.skip(posIn);++ if (indexHasPayloads) {+ // Skip payloadLength block:+ pforUtil.skip(payIn);++ // Skip payloadBytes block:+ int numBytes = payIn.readVInt();+ payIn.seek(payIn.getFilePointer() + numBytes);+ }++ if (indexHasOffsets) {+ pforUtil.skip(payIn);+ pforUtil.skip(payIn);+ }+ toSkip -= BLOCK_SIZE;+ }+ refillPositions();+ payloadByteUpto = 0;+ posBufferUpto = 0;+ while (posBufferUpto < toSkip) {+ if (indexHasPayloads) {+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];+ }+ posBufferUpto++;+ }+ }++ position = 0;+ lastStartOffset = 0;+ }++ @Override+ public int nextPosition() throws IOException {+ assert posPendingCount > 0;++ if (posPendingFP != -1) {+ posIn.seek(posPendingFP);+ posPendingFP = -1;++ if (payPendingFP != -1 && payIn != null) {+ payIn.seek(payPendingFP);+ payPendingFP = -1;+ }++ // Force buffer refill:+ posBufferUpto = BLOCK_SIZE;+ }++ if (posPendingCount > freq) {+ skipPositions();+ posPendingCount = freq;+ }++ if (posBufferUpto == BLOCK_SIZE) {+ refillPositions();+ posBufferUpto = 0;+ }+ position += posDeltaBuffer[posBufferUpto];++ if (indexHasPayloads) {+ payloadLength = (int) payloadLengthBuffer[posBufferUpto];+ payload.bytes = payloadBytes;+ payload.offset = payloadByteUpto;+ payload.length = payloadLength;+ payloadByteUpto += payloadLength;+ }++ if (indexHasOffsets) {+ startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];+ endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];+ lastStartOffset = startOffset;+ }++ posBufferUpto++;+ posPendingCount--;+ return position;+ }++ @Override+ public int startOffset() {+ return startOffset;+ }++ @Override+ public int endOffset() {+ return endOffset;+ }++ @Override+ public BytesRef getPayload() {+ if (payloadLength == 0) {+ return null;+ } else {+ return payload;+ }+ }++ @Override+ public long cost() {+ return docFreq;+ }+ }++ final class BlockImpactsDocsEnum extends ImpactsEnum {++ final ForUtil forUtil = new ForUtil();+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);+ final PForUtil pforUtil = new PForUtil(forUtil);++ private final long[] docBuffer = new long[BLOCK_SIZE + 1];+ private final long[] freqBuffer = new long[BLOCK_SIZE];++ private int docBufferUpto;++ private final Lucene99ScoreSkipReader skipper;++ final IndexInput docIn;++ final boolean indexHasFreqs;++ private int docFreq; // number of docs in this posting list+ private int blockUpto; // number of documents in or before the current block+ private int doc; // doc we last read+ private long accum; // accumulator for doc deltas++ private int nextSkipDoc = -1;++ private long seekTo = -1;++ // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block+ // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer+ // (needsFreq=false)+ private boolean isFreqsRead;++ public BlockImpactsDocsEnum(FieldInfo fieldInfo, IntBlockTermState termState)+ throws IOException {+ indexHasFreqs = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;+ final boolean indexHasPositions =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;+ final boolean indexHasOffsets =+ fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ final boolean indexHasPayloads = fieldInfo.hasPayloads();++ this.docIn = Lucene99PostingsReader.this.docIn.clone();++ docFreq = termState.docFreq;+ docIn.seek(termState.docStartFP);++ doc = -1;+ accum = 0;+ blockUpto = 0;+ docBufferUpto = BLOCK_SIZE;++ skipper =+ new Lucene99ScoreSkipReader(+ docIn.clone(), MAX_SKIP_LEVELS, indexHasPositions, indexHasOffsets, indexHasPayloads);+ skipper.init(+ termState.docStartFP + termState.skipOffset,+ termState.docStartFP,+ termState.posStartFP,+ termState.payStartFP,+ docFreq);++ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in+ // advance()+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;+ this.isFreqsRead = true;+ if (indexHasFreqs == false) {+ Arrays.fill(freqBuffer, 1L);+ }+ }++ @Override+ public int freq() throws IOException {+ if (isFreqsRead == false) {+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block+ isFreqsRead = true;+ }+ return (int) freqBuffer[docBufferUpto - 1];+ }++ @Override+ public int docID() {+ return doc;+ }++ private void refillDocs() throws IOException {+ // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn+ // after it+ if (isFreqsRead == false) {+ pforUtil.skip(docIn);+ isFreqsRead = true;+ }++ final int left = docFreq - blockUpto;+ assert left >= 0;++ if (left >= BLOCK_SIZE) {+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);+ if (indexHasFreqs) {+ isFreqsRead = false;+ }+ blockUpto += BLOCK_SIZE;+ } else {+ readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreqs);+ prefixSum(docBuffer, left, accum);+ docBuffer[left] = NO_MORE_DOCS;+ blockUpto += left;+ }+ accum = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;+ }++ @Override+ public void advanceShallow(int target) throws IOException {+ if (target > nextSkipDoc) {+ // always plus one to fix the result, since skip position in Lucene99SkipReader+ // is a little different from MultiLevelSkipListReader+ final int newDocUpto = skipper.skipTo(target) + 1;++ if (newDocUpto >= blockUpto) {+ // Skipper moved+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;+ blockUpto = newDocUpto;++ // Force to read next block+ docBufferUpto = BLOCK_SIZE;+ accum = skipper.getDoc();+ seekTo = skipper.getDocPointer(); // delay the seek+ }+ // next time we call advance, this is used to+ // foresee whether skipper is necessary.+ nextSkipDoc = skipper.getNextSkipDoc();+ }+ assert nextSkipDoc >= target;+ }++ @Override+ public Impacts getImpacts() throws IOException {+ // nextDoc() doesn't advance skip lists, so it's important to do it here to make sure we're+ // not returning impacts over a bigger range of doc IDs than necessary.+ advanceShallow(doc);+ return skipper.getImpacts();+ }++ @Override+ public int nextDoc() throws IOException {+ if (docBufferUpto == BLOCK_SIZE) {+ if (seekTo >= 0) {+ docIn.seek(seekTo);+ isFreqsRead = true; // reset isFreqsRead+ seekTo = -1;+ }+ refillDocs();+ }+ return this.doc = (int) docBuffer[docBufferUpto++];+ }++ @Override+ public int advance(int target) throws IOException {+ if (target > nextSkipDoc) {+ advanceShallow(target);+ }+ if (docBufferUpto == BLOCK_SIZE) {+ if (seekTo >= 0) {+ docIn.seek(seekTo);+ isFreqsRead = true; // reset isFreqsRead+ seekTo = -1;+ }+ refillDocs();+ }++ int next = findFirstGreater(docBuffer, target, docBufferUpto);+ this.doc = (int) docBuffer[next];+ docBufferUpto = next + 1;+ return doc;+ }++ @Override+ public int nextPosition() throws IOException {+ return -1;+ }++ @Override+ public int startOffset() {+ return -1;+ }++ @Override+ public int endOffset() {+ return -1;+ }++ @Override+ public BytesRef getPayload() {+ return null;+ }++ @Override+ public long cost() {+ return docFreq;+ }+ }++ final class BlockImpactsPostingsEnum extends ImpactsEnum {+ final ForUtil forUtil = new ForUtil();+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);+ final PForUtil pforUtil = new PForUtil(forUtil);++ private final long[] docBuffer = new long[BLOCK_SIZE];+ private final long[] freqBuffer = new long[BLOCK_SIZE];+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];++ private int docBufferUpto;+ private int posBufferUpto;++ private final Lucene99ScoreSkipReader skipper;++ final IndexInput docIn;+ final IndexInput posIn;++ final boolean indexHasOffsets;+ final boolean indexHasPayloads;++ private int docFreq; // number of docs in this posting list+ private long totalTermFreq; // number of positions in this posting list+ private int docUpto; // how many docs we've read+ private int doc; // doc we last read+ private long accum; // accumulator for doc deltas+ private int freq; // freq we last read+ private int position; // current position++ // how many positions "behind" we are; nextPosition must+ // skip these to "catch up":+ private int posPendingCount;++ // Lazy pos seek: if != -1 then we must seek to this FP+ // before reading positions:+ private long posPendingFP;++ // Where this term's postings start in the .doc file:+ private long docTermStartFP;++ // Where this term's postings start in the .pos file:+ private long posTermStartFP;++ // Where this term's payloads/offsets start in the .pay+ // file:+ private long payTermStartFP;++ // File pointer where the last (vInt encoded) pos delta+ // block is. We need this to know whether to bulk+ // decode vs vInt decode the block:+ private long lastPosBlockFP;++ private int nextSkipDoc = -1;++ private long seekTo = -1;++ public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)+ throws IOException {+ indexHasOffsets =+ fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ indexHasPayloads = fieldInfo.hasPayloads();++ this.docIn = Lucene99PostingsReader.this.docIn.clone();++ this.posIn = Lucene99PostingsReader.this.posIn.clone();++ docFreq = termState.docFreq;+ docTermStartFP = termState.docStartFP;+ posTermStartFP = termState.posStartFP;+ payTermStartFP = termState.payStartFP;+ totalTermFreq = termState.totalTermFreq;+ docIn.seek(docTermStartFP);+ posPendingFP = posTermStartFP;+ posPendingCount = 0;+ if (termState.totalTermFreq < BLOCK_SIZE) {+ lastPosBlockFP = posTermStartFP;+ } else if (termState.totalTermFreq == BLOCK_SIZE) {+ lastPosBlockFP = -1;+ } else {+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;+ }++ doc = -1;+ accum = 0;+ docUpto = 0;+ docBufferUpto = BLOCK_SIZE;++ skipper =+ new Lucene99ScoreSkipReader(+ docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);+ skipper.init(+ docTermStartFP + termState.skipOffset,+ docTermStartFP,+ posTermStartFP,+ payTermStartFP,+ docFreq);+ }++ @Override+ public int freq() throws IOException {+ return freq;+ }++ @Override+ public int docID() {+ return doc;+ }++ private void refillDocs() throws IOException {+ final int left = docFreq - docUpto;+ assert left >= 0;++ if (left >= BLOCK_SIZE) {+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);+ pforUtil.decode(docIn, freqBuffer);+ } else {+ readVIntBlock(docIn, docBuffer, freqBuffer, left, true);+ prefixSum(docBuffer, left, accum);+ docBuffer[left] = NO_MORE_DOCS;+ }+ accum = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ }++ private void refillPositions() throws IOException {+ if (posIn.getFilePointer() == lastPosBlockFP) {+ final int count = (int) (totalTermFreq % BLOCK_SIZE);+ int payloadLength = 0;+ for (int i = 0; i < count; i++) {+ int code = posIn.readVInt();+ if (indexHasPayloads) {+ if ((code & 1) != 0) {+ payloadLength = posIn.readVInt();+ }+ posDeltaBuffer[i] = code >>> 1;+ if (payloadLength != 0) {+ posIn.seek(posIn.getFilePointer() + payloadLength);+ }+ } else {+ posDeltaBuffer[i] = code;+ }+ if (indexHasOffsets) {+ if ((posIn.readVInt() & 1) != 0) {+ // offset length changed+ posIn.readVInt();+ }+ }+ }+ } else {+ pforUtil.decode(posIn, posDeltaBuffer);+ }+ }++ @Override+ public void advanceShallow(int target) throws IOException {+ if (target > nextSkipDoc) {+ // always plus one to fix the result, since skip position in Lucene99SkipReader+ // is a little different from MultiLevelSkipListReader+ final int newDocUpto = skipper.skipTo(target) + 1;++ if (newDocUpto > docUpto) {+ // Skipper moved+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;+ docUpto = newDocUpto;++ // Force to read next block+ docBufferUpto = BLOCK_SIZE;+ accum = skipper.getDoc();+ posPendingFP = skipper.getPosPointer();+ posPendingCount = skipper.getPosBufferUpto();+ seekTo = skipper.getDocPointer(); // delay the seek+ }+ // next time we call advance, this is used to+ // foresee whether skipper is necessary.+ nextSkipDoc = skipper.getNextSkipDoc();+ }+ assert nextSkipDoc >= target;+ }++ @Override+ public Impacts getImpacts() throws IOException {+ advanceShallow(doc);+ return skipper.getImpacts();+ }++ @Override+ public int nextDoc() throws IOException {+ return advance(doc + 1);+ }++ @Override+ public int advance(int target) throws IOException {+ if (target > nextSkipDoc) {+ advanceShallow(target);+ }+ if (docBufferUpto == BLOCK_SIZE) {+ if (seekTo >= 0) {+ docIn.seek(seekTo);+ seekTo = -1;+ }+ refillDocs();+ }++ int next = findFirstGreater(docBuffer, target, docBufferUpto);+ if (next == BLOCK_SIZE) {+ return doc = NO_MORE_DOCS;+ }+ this.doc = (int) docBuffer[next];+ this.freq = (int) freqBuffer[next];+ for (int i = docBufferUpto; i <= next; ++i) {+ posPendingCount += freqBuffer[i];+ }+ docUpto += next - docBufferUpto + 1;+ docBufferUpto = next + 1;+ position = 0;+ return doc;+ }++ // TODO: in theory we could avoid loading frq block+ // when not needed, ie, use skip data to load how far to+ // seek the pos pointer ... instead of having to load frq+ // blocks only to sum up how many positions to skip+ private void skipPositions() throws IOException {+ // Skip positions now:+ int toSkip = posPendingCount - freq;++ final int leftInBlock = BLOCK_SIZE - posBufferUpto;+ if (toSkip < leftInBlock) {+ posBufferUpto += toSkip;+ } else {+ toSkip -= leftInBlock;+ while (toSkip >= BLOCK_SIZE) {+ assert posIn.getFilePointer() != lastPosBlockFP;+ pforUtil.skip(posIn);+ toSkip -= BLOCK_SIZE;+ }+ refillPositions();+ posBufferUpto = toSkip;+ }++ position = 0;+ }++ @Override+ public int nextPosition() throws IOException {+ assert posPendingCount > 0;++ if (posPendingFP != -1) {+ posIn.seek(posPendingFP);+ posPendingFP = -1;++ // Force buffer refill:+ posBufferUpto = BLOCK_SIZE;+ }++ if (posPendingCount > freq) {+ skipPositions();+ posPendingCount = freq;+ }++ if (posBufferUpto == BLOCK_SIZE) {+ refillPositions();+ posBufferUpto = 0;+ }+ position += posDeltaBuffer[posBufferUpto++];++ posPendingCount--;+ return position;+ }++ @Override+ public int startOffset() {+ return -1;+ }++ @Override+ public int endOffset() {+ return -1;+ }++ @Override+ public BytesRef getPayload() {+ return null;+ }++ @Override+ public long cost() {+ return docFreq;+ }+ }++ final class BlockImpactsEverythingEnum extends ImpactsEnum {++ final ForUtil forUtil = new ForUtil();+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);+ final PForUtil pforUtil = new PForUtil(forUtil);++ private final long[] docBuffer = new long[BLOCK_SIZE];+ private final long[] freqBuffer = new long[BLOCK_SIZE];+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];++ private final long[] payloadLengthBuffer;+ private final long[] offsetStartDeltaBuffer;+ private final long[] offsetLengthBuffer;++ private byte[] payloadBytes;+ private int payloadByteUpto;+ private int payloadLength;++ private int lastStartOffset;+ private int startOffset = -1;+ private int endOffset = -1;++ private int docBufferUpto;+ private int posBufferUpto;++ private final Lucene99ScoreSkipReader skipper;++ final IndexInput docIn;+ final IndexInput posIn;+ final IndexInput payIn;+ final BytesRef payload;++ final boolean indexHasFreq;+ final boolean indexHasPos;+ final boolean indexHasOffsets;+ final boolean indexHasPayloads;++ private int docFreq; // number of docs in this posting list+ private long totalTermFreq; // number of positions in this posting list+ private int docUpto; // how many docs we've read+ private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads+ private int doc; // doc we last read+ private long accum; // accumulator for doc deltas+ private int position; // current position++ // how many positions "behind" we are; nextPosition must+ // skip these to "catch up":+ private int posPendingCount;++ // Lazy pos seek: if != -1 then we must seek to this FP+ // before reading positions:+ private long posPendingFP;++ // Lazy pay seek: if != -1 then we must seek to this FP+ // before reading payloads/offsets:+ private long payPendingFP;++ // Where this term's postings start in the .doc file:+ private long docTermStartFP;++ // Where this term's postings start in the .pos file:+ private long posTermStartFP;++ // Where this term's payloads/offsets start in the .pay+ // file:+ private long payTermStartFP;++ // File pointer where the last (vInt encoded) pos delta+ // block is. We need this to know whether to bulk+ // decode vs vInt decode the block:+ private long lastPosBlockFP;++ private int nextSkipDoc = -1;++ private final boolean needsPositions;+ private final boolean needsOffsets; // true if we actually need offsets+ private final boolean needsPayloads; // true if we actually need payloads++ private boolean+ isFreqsRead; // shows if freqBuffer for the current doc block are read into freqBuffer++ private long seekTo = -1;++ public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags)+ throws IOException {+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;+ indexHasPos =+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;+ indexHasOffsets =+ fieldInfo+ .getIndexOptions()+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)+ >= 0;+ indexHasPayloads = fieldInfo.hasPayloads();++ needsPositions = PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);+ needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);+ needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);++ this.docIn = Lucene99PostingsReader.this.docIn.clone();++ if (indexHasPos && needsPositions) {+ this.posIn = Lucene99PostingsReader.this.posIn.clone();+ } else {+ this.posIn = null;+ }++ if ((indexHasOffsets && needsOffsets) || (indexHasPayloads && needsPayloads)) {+ this.payIn = Lucene99PostingsReader.this.payIn.clone();+ } else {+ this.payIn = null;+ }++ if (indexHasOffsets) {+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];+ offsetLengthBuffer = new long[BLOCK_SIZE];+ } else {+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ startOffset = -1;+ endOffset = -1;+ }++ if (indexHasPayloads) {+ payloadLengthBuffer = new long[BLOCK_SIZE];+ payloadBytes = new byte[128];+ payload = new BytesRef();+ } else {+ payloadLengthBuffer = null;+ payloadBytes = null;+ payload = null;+ }++ docFreq = termState.docFreq;+ docTermStartFP = termState.docStartFP;+ posTermStartFP = termState.posStartFP;+ payTermStartFP = termState.payStartFP;+ totalTermFreq = termState.totalTermFreq;+ docIn.seek(docTermStartFP);+ posPendingFP = posTermStartFP;+ payPendingFP = payTermStartFP;+ posPendingCount = 0;+ if (termState.totalTermFreq < BLOCK_SIZE) {+ lastPosBlockFP = posTermStartFP;+ } else if (termState.totalTermFreq == BLOCK_SIZE) {+ lastPosBlockFP = -1;+ } else {+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;+ }++ doc = -1;+ accum = 0;+ docUpto = 0;+ posDocUpTo = 0;+ isFreqsRead = true;+ docBufferUpto = BLOCK_SIZE;++ skipper =+ new Lucene99ScoreSkipReader(+ docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);+ skipper.init(+ docTermStartFP + termState.skipOffset,+ docTermStartFP,+ posTermStartFP,+ payTermStartFP,+ docFreq);++ if (indexHasFreq == false) {+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ freqBuffer[i] = 1;+ }+ }+ }++ @Override+ public int freq() throws IOException {+ if (indexHasFreq && (isFreqsRead == false)) {+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block+ isFreqsRead = true;+ }+ return (int) freqBuffer[docBufferUpto - 1];+ }++ @Override+ public int docID() {+ return doc;+ }++ private void refillDocs() throws IOException {+ if (indexHasFreq) {+ if (isFreqsRead == false) { // previous freq block was not read+ // check if we need to load the previous freq block to catch up on positions or we can+ // skip it+ if (indexHasPos && needsPositions && (posDocUpTo < docUpto)) {+ pforUtil.decode(docIn, freqBuffer); // load the previous freq block+ } else {+ pforUtil.skip(docIn); // skip it+ }+ isFreqsRead = true;+ }+ if (indexHasPos && needsPositions) {+ while (posDocUpTo+ < docUpto) { // catch on positions, bring posPendingCount upto the current doc+ posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];+ posDocUpTo++;+ }+ }+ }++ final int left = docFreq - docUpto;+ assert left >= 0;++ if (left >= BLOCK_SIZE) {+ forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);+ if (indexHasFreq) {+ isFreqsRead =+ false; // freq block will be loaded lazily when necessary, we don't load it here+ }+ } else {+ readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);+ prefixSum(docBuffer, left, accum);+ docBuffer[left] = NO_MORE_DOCS;+ }+ accum = docBuffer[BLOCK_SIZE - 1];+ docBufferUpto = 0;+ }++ private void refillPositions() throws IOException {+ if (posIn.getFilePointer() == lastPosBlockFP) {+ final int count = (int) (totalTermFreq % BLOCK_SIZE);+ int payloadLength = 0;+ int offsetLength = 0;+ payloadByteUpto = 0;+ for (int i = 0; i < count; i++) {+ int code = posIn.readVInt();+ if (indexHasPayloads) {+ if ((code & 1) != 0) {+ payloadLength = posIn.readVInt();+ }+ payloadLengthBuffer[i] = payloadLength;+ posDeltaBuffer[i] = code >>> 1;+ if (payloadLength != 0) {+ if (payloadByteUpto + payloadLength > payloadBytes.length) {+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);+ }+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);+ payloadByteUpto += payloadLength;+ }+ } else {+ posDeltaBuffer[i] = code;+ }++ if (indexHasOffsets) {+ int deltaCode = posIn.readVInt();+ if ((deltaCode & 1) != 0) {+ offsetLength = posIn.readVInt();+ }+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;+ offsetLengthBuffer[i] = offsetLength;+ }+ }+ payloadByteUpto = 0;+ } else {+ pforUtil.decode(posIn, posDeltaBuffer);++ if (indexHasPayloads && payIn != null) {+ if (needsPayloads) {+ pforUtil.decode(payIn, payloadLengthBuffer);+ int numBytes = payIn.readVInt();++ if (numBytes > payloadBytes.length) {+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);+ }+ payIn.readBytes(payloadBytes, 0, numBytes);+ } else {+ // this works, because when writing a vint block we always force the first length to be+ // written+ pforUtil.skip(payIn); // skip over lengths+ int numBytes = payIn.readVInt(); // read length of payloadBytes+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes+ }+ payloadByteUpto = 0;+ }++ if (indexHasOffsets && payIn != null) {+ if (needsOffsets) {+ pforUtil.decode(payIn, offsetStartDeltaBuffer);+ pforUtil.decode(payIn, offsetLengthBuffer);+ } else {+ // this works, because when writing a vint block we always force the first length to be+ // written+ pforUtil.skip(payIn); // skip over starts+ pforUtil.skip(payIn); // skip over lengths+ }+ }+ }+ }++ @Override+ public void advanceShallow(int target) throws IOException {+ if (target > nextSkipDoc) {+ // always plus one to fix the result, since skip position in Lucene99SkipReader+ // is a little different from MultiLevelSkipListReader+ final int newDocUpto = skipper.skipTo(target) + 1;++ if (newDocUpto > docUpto) {+ // Skipper moved+ assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;+ docUpto = newDocUpto;+ posDocUpTo = docUpto;++ // Force to read next block+ docBufferUpto = BLOCK_SIZE;+ accum = skipper.getDoc();+ posPendingFP = skipper.getPosPointer();+ payPendingFP = skipper.getPayPointer();+ posPendingCount = skipper.getPosBufferUpto();+ lastStartOffset = 0; // new document+ payloadByteUpto = skipper.getPayloadByteUpto(); // actually, this is just lastSkipEntry+ seekTo = skipper.getDocPointer(); // delay the seek+ }+ // next time we call advance, this is used to+ // foresee whether skipper is necessary.+ nextSkipDoc = skipper.getNextSkipDoc();+ }+ assert nextSkipDoc >= target;+ }++ @Override+ public Impacts getImpacts() throws IOException {+ advanceShallow(doc);+ return skipper.getImpacts();+ }++ @Override+ public int nextDoc() throws IOException {+ return advance(doc + 1);+ }++ @Override+ public int advance(int target) throws IOException {+ if (target > nextSkipDoc) {+ advanceShallow(target);+ }+ if (docBufferUpto == BLOCK_SIZE) {+ if (seekTo >= 0) {+ docIn.seek(seekTo);+ seekTo = -1;+ isFreqsRead = true; // reset isFreqsRead+ }+ refillDocs();+ }++ // Now scan:+ long doc;+ while (true) {+ doc = docBuffer[docBufferUpto];+ docBufferUpto++;+ docUpto++;++ if (doc >= target) {+ break;+ }++ if (docBufferUpto == BLOCK_SIZE) {+ return this.doc = NO_MORE_DOCS;+ }+ }+ position = 0;+ lastStartOffset = 0;++ return this.doc = (int) doc;+ }++ // TODO: in theory we could avoid loading frq block+ // when not needed, ie, use skip data to load how far to+ // seek the pos pointer ... instead of having to load frq+ // blocks only to sum up how many positions to skip+ private void skipPositions() throws IOException {+ // Skip positions now:+ int toSkip = posPendingCount - (int) freqBuffer[docBufferUpto - 1];+ // if (DEBUG) {+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);+ // }++ final int leftInBlock = BLOCK_SIZE - posBufferUpto;+ if (toSkip < leftInBlock) {+ int end = posBufferUpto + toSkip;+ while (posBufferUpto < end) {+ if (indexHasPayloads) {+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];+ }+ posBufferUpto++;+ }+ } else {+ toSkip -= leftInBlock;+ while (toSkip >= BLOCK_SIZE) {+ assert posIn.getFilePointer() != lastPosBlockFP;+ pforUtil.skip(posIn);++ if (indexHasPayloads && payIn != null) {+ // Skip payloadLength block:+ pforUtil.skip(payIn);++ // Skip payloadBytes block:+ int numBytes = payIn.readVInt();+ payIn.seek(payIn.getFilePointer() + numBytes);+ }++ if (indexHasOffsets && payIn != null) {+ pforUtil.skip(payIn);+ pforUtil.skip(payIn);+ }+ toSkip -= BLOCK_SIZE;+ }+ refillPositions();+ payloadByteUpto = 0;+ posBufferUpto = 0;+ while (posBufferUpto < toSkip) {+ if (indexHasPayloads) {+ payloadByteUpto += payloadLengthBuffer[posBufferUpto];+ }+ posBufferUpto++;+ }+ }++ position = 0;+ lastStartOffset = 0;+ }++ @Override+ public int nextPosition() throws IOException {+ if (indexHasPos == false || needsPositions == false) {+ return -1;+ }++ if (isFreqsRead == false) {+ pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this docs block+ isFreqsRead = true;+ }+ while (posDocUpTo < docUpto) { // bring posPendingCount upto the current doc+ posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];+ posDocUpTo++;+ }++ assert posPendingCount > 0;++ if (posPendingFP != -1) {+ posIn.seek(posPendingFP);+ posPendingFP = -1;++ if (payPendingFP != -1 && payIn != null) {+ payIn.seek(payPendingFP);+ payPendingFP = -1;+ }++ // Force buffer refill:+ posBufferUpto = BLOCK_SIZE;+ }++ if (posPendingCount > freqBuffer[docBufferUpto - 1]) {+ skipPositions();+ posPendingCount = (int) freqBuffer[docBufferUpto - 1];+ }++ if (posBufferUpto == BLOCK_SIZE) {+ refillPositions();+ posBufferUpto = 0;+ }+ position += posDeltaBuffer[posBufferUpto];++ if (indexHasPayloads) {+ payloadLength = (int) payloadLengthBuffer[posBufferUpto];+ payload.bytes = payloadBytes;+ payload.offset = payloadByteUpto;+ payload.length = payloadLength;+ payloadByteUpto += payloadLength;+ }++ if (indexHasOffsets && needsOffsets) {+ startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];+ endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];+ lastStartOffset = startOffset;+ }++ posBufferUpto++;+ posPendingCount--;+ return position;+ }++ @Override+ public int startOffset() {+ return startOffset;+ }++ @Override+ public int endOffset() {+ return endOffset;+ }++ @Override+ public BytesRef getPayload() {+ if (payloadLength == 0) {+ return null;+ } else {+ return payload;+ }+ }++ @Override+ public long cost() {+ return docFreq;+ }+ }++ @Override+ public void checkIntegrity() throws IOException {+ if (docIn != null) {+ CodecUtil.checksumEntireFile(docIn);+ }+ if (posIn != null) {+ CodecUtil.checksumEntireFile(posIn);+ }+ if (payIn != null) {+ CodecUtil.checksumEntireFile(payIn);+ }+ }++ @Override+ public String toString() {+ return getClass().getSimpleName()+ + "(positions="+ + (posIn != null)+ + ",payloads="+ + (payIn != null)+ + ")";+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.javanew file mode 100644index 00000000000..34b4c9c62a3--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java@@ -0,0 +1,539 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;+import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;++import java.io.IOException;+import org.apache.lucene.codecs.BlockTermState;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;+import org.apache.lucene.codecs.PushPostingsWriterBase;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BitUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;++/**+ * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.+ *+ * <p>Postings list for each term will be stored separately.+ *+ * @see Lucene99SkipWriter for details about skipping setting and postings layout.+ * @lucene.experimental+ */+public final class Lucene99PostingsWriter extends PushPostingsWriterBase {++ IndexOutput docOut;+ IndexOutput posOut;+ IndexOutput payOut;++ static final IntBlockTermState emptyState = new IntBlockTermState();+ IntBlockTermState lastState;++ // Holds starting file pointers for current term:+ private long docStartFP;+ private long posStartFP;+ private long payStartFP;++ final long[] docDeltaBuffer;+ final long[] freqBuffer;+ private int docBufferUpto;++ final long[] posDeltaBuffer;+ final long[] payloadLengthBuffer;+ final long[] offsetStartDeltaBuffer;+ final long[] offsetLengthBuffer;+ private int posBufferUpto;++ private byte[] payloadBytes;+ private int payloadByteUpto;++ private int lastBlockDocID;+ private long lastBlockPosFP;+ private long lastBlockPayFP;+ private int lastBlockPosBufferUpto;+ private int lastBlockPayloadByteUpto;++ private int lastDocID;+ private int lastPosition;+ private int lastStartOffset;+ private int docCount;++ private final PForUtil pforUtil;+ private final ForDeltaUtil forDeltaUtil;+ private final Lucene99SkipWriter skipWriter;++ private boolean fieldHasNorms;+ private NumericDocValues norms;+ private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =+ new CompetitiveImpactAccumulator();++ /** Creates a postings writer */+ public Lucene99PostingsWriter(SegmentWriteState state) throws IOException {++ String docFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.DOC_EXTENSION);+ docOut = state.directory.createOutput(docFileName, state.context);+ IndexOutput posOut = null;+ IndexOutput payOut = null;+ boolean success = false;+ try {+ CodecUtil.writeIndexHeader(+ docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);+ final ForUtil forUtil = new ForUtil();+ forDeltaUtil = new ForDeltaUtil(forUtil);+ pforUtil = new PForUtil(forUtil);+ if (state.fieldInfos.hasProx()) {+ posDeltaBuffer = new long[BLOCK_SIZE];+ String posFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.POS_EXTENSION);+ posOut = state.directory.createOutput(posFileName, state.context);+ CodecUtil.writeIndexHeader(+ posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);++ if (state.fieldInfos.hasPayloads()) {+ payloadBytes = new byte[128];+ payloadLengthBuffer = new long[BLOCK_SIZE];+ } else {+ payloadBytes = null;+ payloadLengthBuffer = null;+ }++ if (state.fieldInfos.hasOffsets()) {+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];+ offsetLengthBuffer = new long[BLOCK_SIZE];+ } else {+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ }++ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {+ String payFileName =+ IndexFileNames.segmentFileName(+ state.segmentInfo.name,+ state.segmentSuffix,+ Lucene99PostingsFormat.PAY_EXTENSION);+ payOut = state.directory.createOutput(payFileName, state.context);+ CodecUtil.writeIndexHeader(+ payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);+ }+ } else {+ posDeltaBuffer = null;+ payloadLengthBuffer = null;+ offsetStartDeltaBuffer = null;+ offsetLengthBuffer = null;+ payloadBytes = null;+ }+ this.payOut = payOut;+ this.posOut = posOut;+ success = true;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(docOut, posOut, payOut);+ }+ }++ docDeltaBuffer = new long[BLOCK_SIZE];+ freqBuffer = new long[BLOCK_SIZE];++ // TODO: should we try skipping every 2/4 blocks...?+ skipWriter =+ new Lucene99SkipWriter(+ MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);+ }++ @Override+ public IntBlockTermState newTermState() {+ return new IntBlockTermState();+ }++ @Override+ public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {+ CodecUtil.writeIndexHeader(+ termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);+ termsOut.writeVInt(BLOCK_SIZE);+ }++ @Override+ public void setField(FieldInfo fieldInfo) {+ super.setField(fieldInfo);+ skipWriter.setField(writePositions, writeOffsets, writePayloads);+ lastState = emptyState;+ fieldHasNorms = fieldInfo.hasNorms();+ }++ @Override+ public void startTerm(NumericDocValues norms) {+ docStartFP = docOut.getFilePointer();+ if (writePositions) {+ posStartFP = posOut.getFilePointer();+ if (writePayloads || writeOffsets) {+ payStartFP = payOut.getFilePointer();+ }+ }+ lastDocID = 0;+ lastBlockDocID = -1;+ skipWriter.resetSkip();+ this.norms = norms;+ competitiveFreqNormAccumulator.clear();+ }++ @Override+ public void startDoc(int docID, int termDocFreq) throws IOException {+ // Have collected a block of docs, and get a new doc.+ // Should write skip data as well as postings list for+ // current block.+ if (lastBlockDocID != -1 && docBufferUpto == 0) {+ skipWriter.bufferSkip(+ lastBlockDocID,+ competitiveFreqNormAccumulator,+ docCount,+ lastBlockPosFP,+ lastBlockPayFP,+ lastBlockPosBufferUpto,+ lastBlockPayloadByteUpto);+ competitiveFreqNormAccumulator.clear();+ }++ final int docDelta = docID - lastDocID;++ if (docID < 0 || (docCount > 0 && docDelta <= 0)) {+ throw new CorruptIndexException(+ "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);+ }++ docDeltaBuffer[docBufferUpto] = docDelta;+ if (writeFreqs) {+ freqBuffer[docBufferUpto] = termDocFreq;+ }++ docBufferUpto++;+ docCount++;++ if (docBufferUpto == BLOCK_SIZE) {+ forDeltaUtil.encodeDeltas(docDeltaBuffer, docOut);+ if (writeFreqs) {+ pforUtil.encode(freqBuffer, docOut);+ }+ // NOTE: don't set docBufferUpto back to 0 here;+ // finishDoc will do so (because it needs to see that+ // the block was filled so it can save skip data)+ }++ lastDocID = docID;+ lastPosition = 0;+ lastStartOffset = 0;++ long norm;+ if (fieldHasNorms) {+ boolean found = norms.advanceExact(docID);+ if (found == false) {+ // This can happen if indexing hits a problem after adding a doc to the+ // postings but before buffering the norm. Such documents are written+ // deleted and will go away on the first merge.+ norm = 1L;+ } else {+ norm = norms.longValue();+ assert norm != 0 : docID;+ }+ } else {+ norm = 1L;+ }++ competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);+ }++ @Override+ public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)+ throws IOException {+ if (position > IndexWriter.MAX_POSITION) {+ throw new CorruptIndexException(+ "position="+ + position+ + " is too large (> IndexWriter.MAX_POSITION="+ + IndexWriter.MAX_POSITION+ + ")",+ docOut);+ }+ if (position < 0) {+ throw new CorruptIndexException("position=" + position + " is < 0", docOut);+ }+ posDeltaBuffer[posBufferUpto] = position - lastPosition;+ if (writePayloads) {+ if (payload == null || payload.length == 0) {+ // no payload+ payloadLengthBuffer[posBufferUpto] = 0;+ } else {+ payloadLengthBuffer[posBufferUpto] = payload.length;+ if (payloadByteUpto + payload.length > payloadBytes.length) {+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);+ }+ System.arraycopy(+ payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);+ payloadByteUpto += payload.length;+ }+ }++ if (writeOffsets) {+ assert startOffset >= lastStartOffset;+ assert endOffset >= startOffset;+ offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;+ offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;+ lastStartOffset = startOffset;+ }++ posBufferUpto++;+ lastPosition = position;+ if (posBufferUpto == BLOCK_SIZE) {+ pforUtil.encode(posDeltaBuffer, posOut);++ if (writePayloads) {+ pforUtil.encode(payloadLengthBuffer, payOut);+ payOut.writeVInt(payloadByteUpto);+ payOut.writeBytes(payloadBytes, 0, payloadByteUpto);+ payloadByteUpto = 0;+ }+ if (writeOffsets) {+ pforUtil.encode(offsetStartDeltaBuffer, payOut);+ pforUtil.encode(offsetLengthBuffer, payOut);+ }+ posBufferUpto = 0;+ }+ }++ @Override+ public void finishDoc() throws IOException {+ // Since we don't know df for current term, we had to buffer+ // those skip data for each block, and when a new doc comes,+ // write them to skip file.+ if (docBufferUpto == BLOCK_SIZE) {+ lastBlockDocID = lastDocID;+ if (posOut != null) {+ if (payOut != null) {+ lastBlockPayFP = payOut.getFilePointer();+ }+ lastBlockPosFP = posOut.getFilePointer();+ lastBlockPosBufferUpto = posBufferUpto;+ lastBlockPayloadByteUpto = payloadByteUpto;+ }+ docBufferUpto = 0;+ }+ }++ /** Called when we are done adding docs to this term */+ @Override+ public void finishTerm(BlockTermState _state) throws IOException {+ IntBlockTermState state = (IntBlockTermState) _state;+ assert state.docFreq > 0;++ // TODO: wasteful we are counting this (counting # docs+ // for this term) in two places?+ assert state.docFreq == docCount : state.docFreq + " vs " + docCount;++ // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to+ // it.+ final int singletonDocID;+ if (state.docFreq == 1) {+ // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq+ singletonDocID = (int) docDeltaBuffer[0];+ } else {+ singletonDocID = -1;+ // vInt encode the remaining doc deltas and freqs:+ for (int i = 0; i < docBufferUpto; i++) {+ final int docDelta = (int) docDeltaBuffer[i];+ final int freq = (int) freqBuffer[i];+ if (!writeFreqs) {+ docOut.writeVInt(docDelta);+ } else if (freq == 1) {+ docOut.writeVInt((docDelta << 1) | 1);+ } else {+ docOut.writeVInt(docDelta << 1);+ docOut.writeVInt(freq);+ }+ }+ }++ final long lastPosBlockOffset;++ if (writePositions) {+ // totalTermFreq is just total number of positions(or payloads, or offsets)+ // associated with current term.+ assert state.totalTermFreq != -1;+ if (state.totalTermFreq > BLOCK_SIZE) {+ // record file offset for last pos in last block+ lastPosBlockOffset = posOut.getFilePointer() - posStartFP;+ } else {+ lastPosBlockOffset = -1;+ }+ if (posBufferUpto > 0) {+ // TODO: should we send offsets/payloads to+ // .pay...? seems wasteful (have to store extra+ // vLong for low (< BLOCK_SIZE) DF terms = vast vast+ // majority)++ // vInt encode the remaining positions/payloads/offsets:+ int lastPayloadLength = -1; // force first payload length to be written+ int lastOffsetLength = -1; // force first offset length to be written+ int payloadBytesReadUpto = 0;+ for (int i = 0; i < posBufferUpto; i++) {+ final int posDelta = (int) posDeltaBuffer[i];+ if (writePayloads) {+ final int payloadLength = (int) payloadLengthBuffer[i];+ if (payloadLength != lastPayloadLength) {+ lastPayloadLength = payloadLength;+ posOut.writeVInt((posDelta << 1) | 1);+ posOut.writeVInt(payloadLength);+ } else {+ posOut.writeVInt(posDelta << 1);+ }++ if (payloadLength != 0) {+ posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);+ payloadBytesReadUpto += payloadLength;+ }+ } else {+ posOut.writeVInt(posDelta);+ }++ if (writeOffsets) {+ int delta = (int) offsetStartDeltaBuffer[i];+ int length = (int) offsetLengthBuffer[i];+ if (length == lastOffsetLength) {+ posOut.writeVInt(delta << 1);+ } else {+ posOut.writeVInt(delta << 1 | 1);+ posOut.writeVInt(length);+ lastOffsetLength = length;+ }+ }+ }++ if (writePayloads) {+ assert payloadBytesReadUpto == payloadByteUpto;+ payloadByteUpto = 0;+ }+ }+ } else {+ lastPosBlockOffset = -1;+ }++ long skipOffset;+ if (docCount > BLOCK_SIZE) {+ skipOffset = skipWriter.writeSkip(docOut) - docStartFP;+ } else {+ skipOffset = -1;+ }++ state.docStartFP = docStartFP;+ state.posStartFP = posStartFP;+ state.payStartFP = payStartFP;+ state.singletonDocID = singletonDocID;+ state.skipOffset = skipOffset;+ state.lastPosBlockOffset = lastPosBlockOffset;+ docBufferUpto = 0;+ posBufferUpto = 0;+ lastDocID = 0;+ docCount = 0;+ }++ @Override+ public void encodeTerm(+ DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)+ throws IOException {+ IntBlockTermState state = (IntBlockTermState) _state;+ if (absolute) {+ lastState = emptyState;+ assert lastState.docStartFP == 0;+ }++ if (lastState.singletonDocID != -1+ && state.singletonDocID != -1+ && state.docStartFP == lastState.docStartFP) {+ // With runs of rare values such as ID fields, the increment of pointers in the docs file is+ // often 0.+ // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we+ // encode the delta+ // between consecutive doc IDs to save space.+ final long delta = (long) state.singletonDocID - lastState.singletonDocID;+ out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);+ } else {+ out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);+ if (state.singletonDocID != -1) {+ out.writeVInt(state.singletonDocID);+ }+ }++ if (writePositions) {+ out.writeVLong(state.posStartFP - lastState.posStartFP);+ if (writePayloads || writeOffsets) {+ out.writeVLong(state.payStartFP - lastState.payStartFP);+ }+ }+ if (writePositions) {+ if (state.lastPosBlockOffset != -1) {+ out.writeVLong(state.lastPosBlockOffset);+ }+ }+ if (state.skipOffset != -1) {+ out.writeVLong(state.skipOffset);+ }+ lastState = state;+ }++ @Override+ public void close() throws IOException {+ // TODO: add a finish() at least to PushBase? DV too...?+ boolean success = false;+ try {+ if (docOut != null) {+ CodecUtil.writeFooter(docOut);+ }+ if (posOut != null) {+ CodecUtil.writeFooter(posOut);+ }+ if (payOut != null) {+ CodecUtil.writeFooter(payOut);+ }+ success = true;+ } finally {+ if (success) {+ IOUtils.close(docOut, posOut, payOut);+ } else {+ IOUtils.closeWhileHandlingException(docOut, posOut, payOut);+ }+ docOut = posOut = payOut = null;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.javanew file mode 100644index 00000000000..f2882c085c4--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java@@ -0,0 +1,159 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import java.util.AbstractList;+import java.util.Arrays;+import java.util.List;+import java.util.RandomAccess;+import org.apache.lucene.index.Impact;+import org.apache.lucene.index.Impacts;+import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.ArrayUtil;++final class Lucene99ScoreSkipReader extends Lucene99SkipReader {++ private final byte[][] impactData;+ private final int[] impactDataLength;+ private final ByteArrayDataInput badi = new ByteArrayDataInput();+ private final Impacts impacts;+ private int numLevels = 1;+ private final MutableImpactList[] perLevelImpacts;++ public Lucene99ScoreSkipReader(+ IndexInput skipStream,+ int maxSkipLevels,+ boolean hasPos,+ boolean hasOffsets,+ boolean hasPayloads) {+ super(skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);+ this.impactData = new byte[maxSkipLevels][];+ Arrays.fill(impactData, new byte[0]);+ this.impactDataLength = new int[maxSkipLevels];+ this.perLevelImpacts = new MutableImpactList[maxSkipLevels];+ for (int i = 0; i < perLevelImpacts.length; ++i) {+ perLevelImpacts[i] = new MutableImpactList();+ }+ impacts =+ new Impacts() {++ @Override+ public int numLevels() {+ return numLevels;+ }++ @Override+ public int getDocIdUpTo(int level) {+ return skipDoc[level];+ }++ @Override+ public List<Impact> getImpacts(int level) {+ assert level < numLevels;+ if (impactDataLength[level] > 0) {+ badi.reset(impactData[level], 0, impactDataLength[level]);+ perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);+ impactDataLength[level] = 0;+ }+ return perLevelImpacts[level];+ }+ };+ }++ @Override+ public int skipTo(int target) throws IOException {+ int result = super.skipTo(target);+ if (numberOfSkipLevels > 0) {+ numLevels = numberOfSkipLevels;+ } else {+ // End of postings don't have skip data anymore, so we fill with dummy data+ // like SlowImpactsEnum.+ numLevels = 1;+ perLevelImpacts[0].length = 1;+ perLevelImpacts[0].impacts[0].freq = Integer.MAX_VALUE;+ perLevelImpacts[0].impacts[0].norm = 1L;+ impactDataLength[0] = 0;+ }+ return result;+ }++ Impacts getImpacts() {+ return impacts;+ }++ @Override+ protected void readImpacts(int level, IndexInput skipStream) throws IOException {+ int length = skipStream.readVInt();+ if (impactData[level].length < length) {+ impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];+ }+ skipStream.readBytes(impactData[level], 0, length);+ impactDataLength[level] = length;+ }++ static MutableImpactList readImpacts(ByteArrayDataInput in, MutableImpactList reuse) {+ int maxNumImpacts = in.length(); // at most one impact per byte+ if (reuse.impacts.length < maxNumImpacts) {+ int oldLength = reuse.impacts.length;+ reuse.impacts = ArrayUtil.grow(reuse.impacts, maxNumImpacts);+ for (int i = oldLength; i < reuse.impacts.length; ++i) {+ reuse.impacts[i] = new Impact(Integer.MAX_VALUE, 1L);+ }+ }++ int freq = 0;+ long norm = 0;+ int length = 0;+ while (in.getPosition() < in.length()) {+ int freqDelta = in.readVInt();+ if ((freqDelta & 0x01) != 0) {+ freq += 1 + (freqDelta >>> 1);+ try {+ norm += 1 + in.readZLong();+ } catch (IOException e) {+ throw new RuntimeException(e); // cannot happen on a BADI+ }+ } else {+ freq += 1 + (freqDelta >>> 1);+ norm++;+ }+ Impact impact = reuse.impacts[length];+ impact.freq = freq;+ impact.norm = norm;+ length++;+ }+ reuse.length = length;+ return reuse;+ }++ static class MutableImpactList extends AbstractList<Impact> implements RandomAccess {+ int length = 1;+ Impact[] impacts = new Impact[] {new Impact(Integer.MAX_VALUE, 1L)};++ @Override+ public Impact get(int index) {+ return impacts[index];+ }++ @Override+ public int size() {+ return length;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.javanew file mode 100644index 00000000000..a6cb7c3a811--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java@@ -0,0 +1,206 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.codecs.MultiLevelSkipListReader;+import org.apache.lucene.store.IndexInput;++/**+ * Implements the skip list reader for block postings format that stores positions and payloads.+ *+ * <p>Although this skipper uses MultiLevelSkipListReader as an interface, its definition of skip+ * position will be a little different.+ *+ * <p>For example, when skipInterval = blockSize = 3, df = 2*skipInterval = 6,+ *+ * <pre>+ * 0 1 2 3 4 5+ * d d d d d d (posting list)+ * ^ ^ (skip point in MultiLeveSkipWriter)+ * ^ (skip point in Lucene99SkipWriter)+ * </pre>+ *+ * <p>In this case, MultiLevelSkipListReader will use the last document as a skip point, while+ * Lucene99SkipReader should assume no skip point will comes.+ *+ * <p>If we use the interface directly in Lucene99SkipReader, it may silly try to read another skip+ * data after the only skip point is loaded.+ *+ * <p>To illustrate this, we can call skipTo(d[5]), since skip point d[3] has smaller docId, and+ * numSkipped+blockSize== df, the MultiLevelSkipListReader will assume the skip list isn't exhausted+ * yet, and try to load a non-existed skip point+ *+ * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)+ */+public class Lucene99SkipReader extends MultiLevelSkipListReader {+ private long[] docPointer;+ private long[] posPointer;+ private long[] payPointer;+ private int[] posBufferUpto;+ private int[] payloadByteUpto;++ private long lastPosPointer;+ private long lastPayPointer;+ private int lastPayloadByteUpto;+ private long lastDocPointer;+ private int lastPosBufferUpto;++ public Lucene99SkipReader(+ IndexInput skipStream,+ int maxSkipLevels,+ boolean hasPos,+ boolean hasOffsets,+ boolean hasPayloads) {+ super(skipStream, maxSkipLevels, ForUtil.BLOCK_SIZE, 8);+ docPointer = new long[maxSkipLevels];+ if (hasPos) {+ posPointer = new long[maxSkipLevels];+ posBufferUpto = new int[maxSkipLevels];+ if (hasPayloads) {+ payloadByteUpto = new int[maxSkipLevels];+ } else {+ payloadByteUpto = null;+ }+ if (hasOffsets || hasPayloads) {+ payPointer = new long[maxSkipLevels];+ } else {+ payPointer = null;+ }+ } else {+ posPointer = null;+ }+ }++ /**+ * Trim original docFreq to tell skipReader read proper number of skip points.+ *+ * <p>Since our definition in Lucene99Skip* is a little different from MultiLevelSkip* This+ * trimmed docFreq will prevent skipReader from: 1. silly reading a non-existed skip point after+ * the last block boundary 2. moving into the vInt block+ */+ protected int trim(int df) {+ return df % ForUtil.BLOCK_SIZE == 0 ? df - 1 : df;+ }++ public void init(+ long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)+ throws IOException {+ super.init(skipPointer, trim(df));+ lastDocPointer = docBasePointer;+ lastPosPointer = posBasePointer;+ lastPayPointer = payBasePointer;++ Arrays.fill(docPointer, docBasePointer);+ if (posPointer != null) {+ Arrays.fill(posPointer, posBasePointer);+ if (payPointer != null) {+ Arrays.fill(payPointer, payBasePointer);+ }+ } else {+ assert posBasePointer == 0;+ }+ }++ /**+ * Returns the doc pointer of the doc to which the last call of {@link+ * MultiLevelSkipListReader#skipTo(int)} has skipped.+ */+ public long getDocPointer() {+ return lastDocPointer;+ }++ public long getPosPointer() {+ return lastPosPointer;+ }++ public int getPosBufferUpto() {+ return lastPosBufferUpto;+ }++ public long getPayPointer() {+ return lastPayPointer;+ }++ public int getPayloadByteUpto() {+ return lastPayloadByteUpto;+ }++ public int getNextSkipDoc() {+ return skipDoc[0];+ }++ @Override+ protected void seekChild(int level) throws IOException {+ super.seekChild(level);+ docPointer[level] = lastDocPointer;+ if (posPointer != null) {+ posPointer[level] = lastPosPointer;+ posBufferUpto[level] = lastPosBufferUpto;+ if (payloadByteUpto != null) {+ payloadByteUpto[level] = lastPayloadByteUpto;+ }+ if (payPointer != null) {+ payPointer[level] = lastPayPointer;+ }+ }+ }++ @Override+ protected void setLastSkipData(int level) {+ super.setLastSkipData(level);+ lastDocPointer = docPointer[level];++ if (posPointer != null) {+ lastPosPointer = posPointer[level];+ lastPosBufferUpto = posBufferUpto[level];+ if (payPointer != null) {+ lastPayPointer = payPointer[level];+ }+ if (payloadByteUpto != null) {+ lastPayloadByteUpto = payloadByteUpto[level];+ }+ }+ }++ @Override+ protected int readSkipData(int level, IndexInput skipStream) throws IOException {+ int delta = skipStream.readVInt();+ docPointer[level] += skipStream.readVLong();++ if (posPointer != null) {+ posPointer[level] += skipStream.readVLong();+ posBufferUpto[level] = skipStream.readVInt();++ if (payloadByteUpto != null) {+ payloadByteUpto[level] = skipStream.readVInt();+ }++ if (payPointer != null) {+ payPointer[level] += skipStream.readVLong();+ }+ }+ readImpacts(level, skipStream);+ return delta;+ }++ // The default impl skips impacts+ protected void readImpacts(int level, IndexInput skipStream) throws IOException {+ skipStream.skipBytes(skipStream.readVInt());+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.javanew file mode 100644index 00000000000..ec50e5e5c16--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java@@ -0,0 +1,233 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import java.util.Arrays;+import java.util.Collection;+import org.apache.lucene.codecs.CompetitiveImpactAccumulator;+import org.apache.lucene.codecs.MultiLevelSkipListWriter;+import org.apache.lucene.index.Impact;+import org.apache.lucene.store.ByteBuffersDataOutput;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.store.IndexOutput;++/**+ * Write skip lists with multiple levels, and support skip within block ints.+ *+ * <p>Assume that docFreq = 28, skipInterval = blockSize = 12+ *+ * <pre>+ * | block#0 | | block#1 | |vInts|+ * d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list)+ * ^ ^ (level 0 skip point)+ * </pre>+ *+ * <p>Note that skipWriter will ignore first document in block#0, since it is useless as a skip+ * point. Also, we'll never skip into the vInts block, only record skip data at the start its start+ * point(if it exist).+ *+ * <p>For each skip point, we will record: 1. docID in former position, i.e. for position 12, record+ * docID[11], etc. 2. its related file points(position, payload), 3. related numbers or+ * uptos(position, payload). 4. start offset.+ */+public final class Lucene99SkipWriter extends MultiLevelSkipListWriter {+ private int[] lastSkipDoc;+ private long[] lastSkipDocPointer;+ private long[] lastSkipPosPointer;+ private long[] lastSkipPayPointer;++ private final IndexOutput docOut;+ private final IndexOutput posOut;+ private final IndexOutput payOut;++ private int curDoc;+ private long curDocPointer;+ private long curPosPointer;+ private long curPayPointer;+ private int curPosBufferUpto;+ private int curPayloadByteUpto;+ private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;+ private boolean fieldHasPositions;+ private boolean fieldHasOffsets;+ private boolean fieldHasPayloads;++ public Lucene99SkipWriter(+ int maxSkipLevels,+ int blockSize,+ int docCount,+ IndexOutput docOut,+ IndexOutput posOut,+ IndexOutput payOut) {+ super(blockSize, 8, maxSkipLevels, docCount);+ this.docOut = docOut;+ this.posOut = posOut;+ this.payOut = payOut;++ lastSkipDoc = new int[maxSkipLevels];+ lastSkipDocPointer = new long[maxSkipLevels];+ if (posOut != null) {+ lastSkipPosPointer = new long[maxSkipLevels];+ if (payOut != null) {+ lastSkipPayPointer = new long[maxSkipLevels];+ }+ }+ curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];+ for (int i = 0; i < maxSkipLevels; ++i) {+ curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();+ }+ }++ public void setField(+ boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {+ this.fieldHasPositions = fieldHasPositions;+ this.fieldHasOffsets = fieldHasOffsets;+ this.fieldHasPayloads = fieldHasPayloads;+ }++ // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the+ // skipper+ // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of+ // junk.+ // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we+ // save+ // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.+ private boolean initialized;+ long lastDocFP;+ long lastPosFP;+ long lastPayFP;++ @Override+ public void resetSkip() {+ lastDocFP = docOut.getFilePointer();+ if (fieldHasPositions) {+ lastPosFP = posOut.getFilePointer();+ if (fieldHasOffsets || fieldHasPayloads) {+ lastPayFP = payOut.getFilePointer();+ }+ }+ if (initialized) {+ for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {+ acc.clear();+ }+ }+ initialized = false;+ }++ private void initSkip() {+ if (!initialized) {+ super.resetSkip();+ Arrays.fill(lastSkipDoc, 0);+ Arrays.fill(lastSkipDocPointer, lastDocFP);+ if (fieldHasPositions) {+ Arrays.fill(lastSkipPosPointer, lastPosFP);+ if (fieldHasOffsets || fieldHasPayloads) {+ Arrays.fill(lastSkipPayPointer, lastPayFP);+ }+ }+ // sets of competitive freq,norm pairs should be empty at this point+ assert Arrays.stream(curCompetitiveFreqNorms)+ .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)+ .mapToInt(Collection::size)+ .sum()+ == 0;+ initialized = true;+ }+ }++ /** Sets the values for the current skip data. */+ public void bufferSkip(+ int doc,+ CompetitiveImpactAccumulator competitiveFreqNorms,+ int numDocs,+ long posFP,+ long payFP,+ int posBufferUpto,+ int payloadByteUpto)+ throws IOException {+ initSkip();+ this.curDoc = doc;+ this.curDocPointer = docOut.getFilePointer();+ this.curPosPointer = posFP;+ this.curPayPointer = payFP;+ this.curPosBufferUpto = posBufferUpto;+ this.curPayloadByteUpto = payloadByteUpto;+ this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms);+ bufferSkip(numDocs);+ }++ private final ByteBuffersDataOutput freqNormOut = ByteBuffersDataOutput.newResettableInstance();++ @Override+ protected void writeSkipData(int level, DataOutput skipBuffer) throws IOException {++ int delta = curDoc - lastSkipDoc[level];++ skipBuffer.writeVInt(delta);+ lastSkipDoc[level] = curDoc;++ skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);+ lastSkipDocPointer[level] = curDocPointer;++ if (fieldHasPositions) {++ skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);+ lastSkipPosPointer[level] = curPosPointer;+ skipBuffer.writeVInt(curPosBufferUpto);++ if (fieldHasPayloads) {+ skipBuffer.writeVInt(curPayloadByteUpto);+ }++ if (fieldHasOffsets || fieldHasPayloads) {+ skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);+ lastSkipPayPointer[level] = curPayPointer;+ }+ }++ CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];+ assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;+ if (level + 1 < numberOfSkipLevels) {+ curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);+ }+ writeImpacts(competitiveFreqNorms, freqNormOut);+ skipBuffer.writeVInt(Math.toIntExact(freqNormOut.size()));+ freqNormOut.copyTo(skipBuffer);+ freqNormOut.reset();+ competitiveFreqNorms.clear();+ }++ public static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out)+ throws IOException {+ Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();+ Impact previous = new Impact(0, 0);+ for (Impact impact : impacts) {+ assert impact.freq > previous.freq;+ assert Long.compareUnsigned(impact.norm, previous.norm) > 0;+ int freqDelta = impact.freq - previous.freq - 1;+ long normDelta = impact.norm - previous.norm - 1;+ if (normDelta == 0) {+ // most of time, norm only increases by 1, so we can fold everything in a single byte+ out.writeVInt(freqDelta << 1);+ } else {+ out.writeVInt((freqDelta << 1) | 1);+ out.writeZLong(normDelta);+ }+ previous = impact;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.javanew file mode 100644index 00000000000..93ebb3d0f9d--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java@@ -0,0 +1,143 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.LongHeap;+import org.apache.lucene.util.packed.PackedInts;++/** Utility class to encode sequences of 128 small positive integers. */+final class PForUtil {++ private static final int MAX_EXCEPTIONS = 7;++ // IDENTITY_PLUS_ONE[i] == i + 1+ private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];++ static {+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ IDENTITY_PLUS_ONE[i] = i + 1;+ }+ }++ static boolean allEqual(long[] l) {+ for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {+ if (l[i] != l[0]) {+ return false;+ }+ }+ return true;+ }++ private final ForUtil forUtil;++ PForUtil(ForUtil forUtil) {+ assert ForUtil.BLOCK_SIZE <= 256 : "blocksize must fit in one byte. got " + ForUtil.BLOCK_SIZE;+ this.forUtil = forUtil;+ }++ /** Encode 128 integers from {@code longs} into {@code out}. */+ void encode(long[] longs, DataOutput out) throws IOException {+ // Determine the top MAX_EXCEPTIONS + 1 values+ final LongHeap top = new LongHeap(MAX_EXCEPTIONS + 1);+ for (int i = 0; i <= MAX_EXCEPTIONS; ++i) {+ top.push(longs[i]);+ }+ long topValue = top.top();+ for (int i = MAX_EXCEPTIONS + 1; i < ForUtil.BLOCK_SIZE; ++i) {+ if (longs[i] > topValue) {+ topValue = top.updateTop(longs[i]);+ }+ }++ long max = 0L;+ for (int i = 1; i <= top.size(); ++i) {+ max = Math.max(max, top.get(i));+ }++ final int maxBitsRequired = PackedInts.bitsRequired(max);+ // We store the patch on a byte, so we can't decrease the number of bits required by more than 8+ final int patchedBitsRequired =+ Math.max(PackedInts.bitsRequired(topValue), maxBitsRequired - 8);+ int numExceptions = 0;+ final long maxUnpatchedValue = (1L << patchedBitsRequired) - 1;+ for (int i = 2; i <= top.size(); ++i) {+ if (top.get(i) > maxUnpatchedValue) {+ numExceptions++;+ }+ }+ final byte[] exceptions = new byte[numExceptions * 2];+ if (numExceptions > 0) {+ int exceptionCount = 0;+ for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {+ if (longs[i] > maxUnpatchedValue) {+ exceptions[exceptionCount * 2] = (byte) i;+ exceptions[exceptionCount * 2 + 1] = (byte) (longs[i] >>> patchedBitsRequired);+ longs[i] &= maxUnpatchedValue;+ exceptionCount++;+ }+ }+ assert exceptionCount == numExceptions : exceptionCount + " " + numExceptions;+ }++ if (allEqual(longs) && maxBitsRequired <= 8) {+ for (int i = 0; i < numExceptions; ++i) {+ exceptions[2 * i + 1] =+ (byte) (Byte.toUnsignedLong(exceptions[2 * i + 1]) << patchedBitsRequired);+ }+ out.writeByte((byte) (numExceptions << 5));+ out.writeVLong(longs[0]);+ } else {+ final int token = (numExceptions << 5) | patchedBitsRequired;+ out.writeByte((byte) token);+ forUtil.encode(longs, patchedBitsRequired, out);+ }+ out.writeBytes(exceptions, exceptions.length);+ }++ /** Decode 128 integers into {@code ints}. */+ void decode(DataInput in, long[] longs) throws IOException {+ final int token = Byte.toUnsignedInt(in.readByte());+ final int bitsPerValue = token & 0x1f;+ final int numExceptions = token >>> 5;+ if (bitsPerValue == 0) {+ Arrays.fill(longs, 0, ForUtil.BLOCK_SIZE, in.readVLong());+ } else {+ forUtil.decode(bitsPerValue, in, longs);+ }+ for (int i = 0; i < numExceptions; ++i) {+ longs[Byte.toUnsignedInt(in.readByte())] |=+ Byte.toUnsignedLong(in.readByte()) << bitsPerValue;+ }+ }++ /** Skip 128 integers. */+ void skip(DataInput in) throws IOException {+ final int token = Byte.toUnsignedInt(in.readByte());+ final int bitsPerValue = token & 0x1f;+ final int numExceptions = token >>> 5;+ if (bitsPerValue == 0) {+ in.readVLong();+ in.skipBytes((numExceptions << 1));+ } else {+ in.skipBytes(forUtil.numBytes(bitsPerValue) + (numExceptions << 1));+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.pynew file mode 100644index 00000000000..cd81d493570--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py@@ -0,0 +1,524 @@+#! /usr/bin/env python++# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from math import gcd++"""Code generation for ForUtil.java"""++MAX_SPECIALIZED_BITS_PER_VALUE = 24+OUTPUT_FILE = "ForUtil.java"+PRIMITIVE_SIZE = [8, 16, 32]+HEADER = """// This file has been automatically generated, DO NOT EDIT++/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import java.io.IOException;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.DataOutput;++// Inspired from https://fulmicoton.com/posts/bitpacking/+// Encodes multiple integers in a long to get SIMD-like speedups.+// If bitsPerValue <= 8 then we pack 8 ints per long+// else if bitsPerValue <= 16 we pack 4 ints per long+// else we pack 2 ints per long+final class ForUtil {++ static final int BLOCK_SIZE = 128;+ private static final int BLOCK_SIZE_LOG2 = 7;++ private static long expandMask32(long mask32) {+ return mask32 | (mask32 << 32);+ }++ private static long expandMask16(long mask16) {+ return expandMask32(mask16 | (mask16 << 16));+ }++ private static long expandMask8(long mask8) {+ return expandMask16(mask8 | (mask8 << 8));+ }++ private static long mask32(int bitsPerValue) {+ return expandMask32((1L << bitsPerValue) - 1);+ }++ private static long mask16(int bitsPerValue) {+ return expandMask16((1L << bitsPerValue) - 1);+ }++ private static long mask8(int bitsPerValue) {+ return expandMask8((1L << bitsPerValue) - 1);+ }++ private static void expand8(long[] arr) {+ for (int i = 0; i < 16; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 56) & 0xFFL;+ arr[16 + i] = (l >>> 48) & 0xFFL;+ arr[32 + i] = (l >>> 40) & 0xFFL;+ arr[48 + i] = (l >>> 32) & 0xFFL;+ arr[64 + i] = (l >>> 24) & 0xFFL;+ arr[80 + i] = (l >>> 16) & 0xFFL;+ arr[96 + i] = (l >>> 8) & 0xFFL;+ arr[112 + i] = l & 0xFFL;+ }+ }++ private static void expand8To32(long[] arr) {+ for (int i = 0; i < 16; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 24) & 0x000000FF000000FFL;+ arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;+ arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;+ arr[48 + i] = l & 0x000000FF000000FFL;+ }+ }++ private static void collapse8(long[] arr) {+ for (int i = 0; i < 16; ++i) {+ arr[i] =+ (arr[i] << 56)+ | (arr[16 + i] << 48)+ | (arr[32 + i] << 40)+ | (arr[48 + i] << 32)+ | (arr[64 + i] << 24)+ | (arr[80 + i] << 16)+ | (arr[96 + i] << 8)+ | arr[112 + i];+ }+ }++ private static void expand16(long[] arr) {+ for (int i = 0; i < 32; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 48) & 0xFFFFL;+ arr[32 + i] = (l >>> 32) & 0xFFFFL;+ arr[64 + i] = (l >>> 16) & 0xFFFFL;+ arr[96 + i] = l & 0xFFFFL;+ }+ }++ private static void expand16To32(long[] arr) {+ for (int i = 0; i < 32; ++i) {+ long l = arr[i];+ arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;+ arr[32 + i] = l & 0x0000FFFF0000FFFFL;+ }+ }++ private static void collapse16(long[] arr) {+ for (int i = 0; i < 32; ++i) {+ arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];+ }+ }++ private static void expand32(long[] arr) {+ for (int i = 0; i < 64; ++i) {+ long l = arr[i];+ arr[i] = l >>> 32;+ arr[64 + i] = l & 0xFFFFFFFFL;+ }+ }++ private static void collapse32(long[] arr) {+ for (int i = 0; i < 64; ++i) {+ arr[i] = (arr[i] << 32) | arr[64 + i];+ }+ }++ private static void prefixSum8(long[] arr, long base) {+ expand8To32(arr);+ prefixSum32(arr, base);+ }++ private static void prefixSum16(long[] arr, long base) {+ // We need to move to the next primitive size to avoid overflows+ expand16To32(arr);+ prefixSum32(arr, base);+ }++ private static void prefixSum32(long[] arr, long base) {+ arr[0] += base << 32;+ innerPrefixSum32(arr);+ expand32(arr);+ final long l = arr[BLOCK_SIZE/2-1];+ for (int i = BLOCK_SIZE/2; i < BLOCK_SIZE; ++i) {+ arr[i] += l;+ }+ }++ // For some reason unrolling seems to help+ private static void innerPrefixSum32(long[] arr) {+ arr[1] += arr[0];+ arr[2] += arr[1];+ arr[3] += arr[2];+ arr[4] += arr[3];+ arr[5] += arr[4];+ arr[6] += arr[5];+ arr[7] += arr[6];+ arr[8] += arr[7];+ arr[9] += arr[8];+ arr[10] += arr[9];+ arr[11] += arr[10];+ arr[12] += arr[11];+ arr[13] += arr[12];+ arr[14] += arr[13];+ arr[15] += arr[14];+ arr[16] += arr[15];+ arr[17] += arr[16];+ arr[18] += arr[17];+ arr[19] += arr[18];+ arr[20] += arr[19];+ arr[21] += arr[20];+ arr[22] += arr[21];+ arr[23] += arr[22];+ arr[24] += arr[23];+ arr[25] += arr[24];+ arr[26] += arr[25];+ arr[27] += arr[26];+ arr[28] += arr[27];+ arr[29] += arr[28];+ arr[30] += arr[29];+ arr[31] += arr[30];+ arr[32] += arr[31];+ arr[33] += arr[32];+ arr[34] += arr[33];+ arr[35] += arr[34];+ arr[36] += arr[35];+ arr[37] += arr[36];+ arr[38] += arr[37];+ arr[39] += arr[38];+ arr[40] += arr[39];+ arr[41] += arr[40];+ arr[42] += arr[41];+ arr[43] += arr[42];+ arr[44] += arr[43];+ arr[45] += arr[44];+ arr[46] += arr[45];+ arr[47] += arr[46];+ arr[48] += arr[47];+ arr[49] += arr[48];+ arr[50] += arr[49];+ arr[51] += arr[50];+ arr[52] += arr[51];+ arr[53] += arr[52];+ arr[54] += arr[53];+ arr[55] += arr[54];+ arr[56] += arr[55];+ arr[57] += arr[56];+ arr[58] += arr[57];+ arr[59] += arr[58];+ arr[60] += arr[59];+ arr[61] += arr[60];+ arr[62] += arr[61];+ arr[63] += arr[62];+ }++ private final long[] tmp = new long[BLOCK_SIZE / 2];++ /** Encode 128 integers from {@code longs} into {@code out}. */+ void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {+ final int nextPrimitive;+ final int numLongs;+ if (bitsPerValue <= 8) {+ nextPrimitive = 8;+ numLongs = BLOCK_SIZE / 8;+ collapse8(longs);+ } else if (bitsPerValue <= 16) {+ nextPrimitive = 16;+ numLongs = BLOCK_SIZE / 4;+ collapse16(longs);+ } else {+ nextPrimitive = 32;+ numLongs = BLOCK_SIZE / 2;+ collapse32(longs);+ }++ final int numLongsPerShift = bitsPerValue * 2;+ int idx = 0;+ int shift = nextPrimitive - bitsPerValue;+ for (int i = 0; i < numLongsPerShift; ++i) {+ tmp[i] = longs[idx++] << shift;+ }+ for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {+ for (int i = 0; i < numLongsPerShift; ++i) {+ tmp[i] |= longs[idx++] << shift;+ }+ }++ final int remainingBitsPerLong = shift + bitsPerValue;+ final long maskRemainingBitsPerLong;+ if (nextPrimitive == 8) {+ maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];+ } else if (nextPrimitive == 16) {+ maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];+ } else {+ maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];+ }++ int tmpIdx = 0;+ int remainingBitsPerValue = bitsPerValue;+ while (idx < numLongs) {+ if (remainingBitsPerValue >= remainingBitsPerLong) {+ remainingBitsPerValue -= remainingBitsPerLong;+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;+ if (remainingBitsPerValue == 0) {+ idx++;+ remainingBitsPerValue = bitsPerValue;+ }+ } else {+ final long mask1, mask2;+ if (nextPrimitive == 8) {+ mask1 = MASKS8[remainingBitsPerValue];+ mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];+ } else if (nextPrimitive == 16) {+ mask1 = MASKS16[remainingBitsPerValue];+ mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];+ } else {+ mask1 = MASKS32[remainingBitsPerValue];+ mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];+ }+ tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);+ remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;+ tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;+ }+ }++ for (int i = 0; i < numLongsPerShift; ++i) {+ out.writeLong(tmp[i]);+ }+ }++ /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */+ int numBytes(int bitsPerValue) {+ return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);+ }++ private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)+ throws IOException {+ final int numLongs = bitsPerValue << 1;+ in.readLongs(tmp, 0, numLongs);+ final long mask = MASKS32[bitsPerValue];+ int longsIdx = 0;+ int shift = 32 - bitsPerValue;+ for (; shift >= 0; shift -= bitsPerValue) {+ shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);+ longsIdx += numLongs;+ }+ final int remainingBitsPerLong = shift + bitsPerValue;+ final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];+ int tmpIdx = 0;+ int remainingBits = remainingBitsPerLong;+ for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {+ int b = bitsPerValue - remainingBits;+ long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;+ while (b >= remainingBitsPerLong) {+ b -= remainingBitsPerLong;+ l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;+ }+ if (b > 0) {+ l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];+ remainingBits = remainingBitsPerLong - b;+ } else {+ remainingBits = remainingBitsPerLong;+ }+ longs[longsIdx] = l;+ }+ }++ /**+ * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which+ * generates SIMD instructions for it in order to shift multiple longs at once.+ */+ private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {+ for (int i = 0; i < count; ++i) {+ b[bi + i] = (a[i] >>> shift) & mask;+ }+ }++"""++def writeRemainderWithSIMDOptimize(bpv, next_primitive, remaining_bits_per_long, o, num_values, f):+ iteration = 1+ num_longs = bpv * num_values / remaining_bits_per_long+ while num_longs % 2 == 0 and num_values % 2 == 0:+ num_longs /= 2+ num_values /= 2+ iteration *= 2++ f.write(' shiftLongs(tmp, %d, tmp, 0, 0, MASK%d_%d);\n' % (iteration * num_longs, next_primitive, remaining_bits_per_long))+ f.write(' for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n' %(o, iteration, num_longs, num_values))+ tmp_idx = 0+ b = bpv+ b -= remaining_bits_per_long+ f.write(' long l0 = tmp[tmpIdx + %d] << %d;\n' %(tmp_idx, b))+ tmp_idx += 1+ while b >= remaining_bits_per_long:+ b -= remaining_bits_per_long+ f.write(' l0 |= tmp[tmpIdx + %d] << %d;\n' %(tmp_idx, b))+ tmp_idx += 1+ f.write(' longs[longsIdx + 0] = l0;\n')+ f.write(' }\n')+++def writeRemainder(bpv, next_primitive, remaining_bits_per_long, o, num_values, f):+ iteration = 1+ num_longs = bpv * num_values / remaining_bits_per_long+ while num_longs % 2 == 0 and num_values % 2 == 0:+ num_longs /= 2+ num_values /= 2+ iteration *= 2+ f.write(' for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n' %(o, iteration, num_longs, num_values))+ i = 0+ remaining_bits = 0+ tmp_idx = 0+ for i in range(int(num_values)):+ b = bpv+ if remaining_bits == 0:+ b -= remaining_bits_per_long+ f.write(' long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b))+ else:+ b -= remaining_bits+ f.write(' long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits, b))+ tmp_idx += 1+ while b >= remaining_bits_per_long:+ b -= remaining_bits_per_long+ f.write(' l%d |= (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n' %(i, tmp_idx, next_primitive, remaining_bits_per_long, b))+ tmp_idx += 1+ if b > 0:+ f.write(' l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n' %(i, tmp_idx, remaining_bits_per_long-b, next_primitive, b))+ remaining_bits = remaining_bits_per_long-b+ f.write(' longs[longsIdx + %d] = l%d;\n' %(i, i))+ f.write(' }\n')+++def writeDecode(bpv, f):+ next_primitive = 32+ if bpv <= 8:+ next_primitive = 8+ elif bpv <= 16:+ next_primitive = 16+ f.write(' private static void decode%d(DataInput in, long[] tmp, long[] longs) throws IOException {\n' %bpv)+ num_values_per_long = 64 / next_primitive+ if bpv == next_primitive:+ f.write(' in.readLongs(longs, 0, %d);\n' %(bpv*2))+ else:+ f.write(' in.readLongs(tmp, 0, %d);\n' %(bpv*2))+ shift = next_primitive - bpv+ o = 0+ while shift >= 0:+ f.write(' shiftLongs(tmp, %d, longs, %d, %d, MASK%d_%d);\n' %(bpv*2, o, shift, next_primitive, bpv))+ o += bpv*2+ shift -= bpv+ if shift + bpv > 0:+ if bpv % (next_primitive % bpv) == 0:+ writeRemainderWithSIMDOptimize(bpv, next_primitive, shift + bpv, o, 128/num_values_per_long - o, f)+ else:+ writeRemainder(bpv, next_primitive, shift + bpv, o, 128/num_values_per_long - o, f)+ f.write(' }\n')+++if __name__ == '__main__':+ f = open(OUTPUT_FILE, 'w')+ f.write(HEADER)+ for primitive_size in PRIMITIVE_SIZE:+ f.write(' private static final long[] MASKS%d = new long[%d];\n' %(primitive_size, primitive_size))+ f.write('\n')+ f.write(' static {\n')+ for primitive_size in PRIMITIVE_SIZE:+ f.write(' for (int i = 0; i < %d; ++i) {\n' %primitive_size)+ f.write(' MASKS%d[i] = mask%d(i);\n' %(primitive_size, primitive_size))+ f.write(' }\n')+ f.write(' }')+ f.write("""+ // mark values in array as final longs to avoid the cost of reading array, arrays should only be+ // used when the idx is a variable+""")+ for primitive_size in PRIMITIVE_SIZE:+ for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)):+ if bpv * 2 != primitive_size or primitive_size == 8:+ f.write(' private static final long MASK%d_%d = MASKS%d[%d];\n' %(primitive_size, bpv, primitive_size, bpv))++ f.write("""+ /** Decode 128 integers into {@code longs}. */+ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {+ switch (bitsPerValue) {+""")+ for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):+ next_primitive = 32+ if bpv <= 8:+ next_primitive = 8+ elif bpv <= 16:+ next_primitive = 16+ f.write(' case %d:\n' %bpv)+ f.write(' decode%d(in, tmp, longs);\n' %bpv)+ f.write(' expand%d(longs);\n' %next_primitive)+ f.write(' break;\n')+ f.write(' default:\n')+ f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n')+ f.write(' expand32(longs);\n')+ f.write(' break;\n')+ f.write(' }\n')+ f.write(' }\n')++ f.write("""+ /**+ * Delta-decode 128 integers into {@code longs}.+ */+ void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs) throws IOException {+ switch (bitsPerValue) {+""")+ for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):+ next_primitive = 32+ if bpv <= 8:+ next_primitive = 8+ elif bpv <= 16:+ next_primitive = 16+ f.write(' case %d:\n' %bpv)+ f.write(' decode%d(in, tmp, longs);\n' %bpv)+ f.write(' prefixSum%d(longs, base);\n' %next_primitive)+ f.write(' break;\n')+ f.write(' default:\n')+ f.write(' decodeSlow(bitsPerValue, in, tmp, longs);\n')+ f.write(' prefixSum32(longs, base);\n')+ f.write(' break;\n')+ f.write(' }\n')+ f.write(' }\n')++ f.write('\n')+ for i in range(1, MAX_SPECIALIZED_BITS_PER_VALUE+1):+ writeDecode(i, f)+ if i < MAX_SPECIALIZED_BITS_PER_VALUE:+ f.write('\n')++ f.write('}\n')diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.javaindex 2511e3be0da..2df2cfc9a30 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java@@ -151,15 +151,15 @@* field names. These are used to store auxiliary information about the document, such as its* title, url, or an identifier to access a database. The set of stored fields are what is* returned for each hit when searching. This is keyed by document number.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term dictionary}. A* dictionary containing all of the terms used in all of the indexed fields of all of the* documents. The dictionary also contains the number of documents which contain the term, and* pointers to the term's frequency and proximity data.- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Frequency data}. For* each term in the dictionary, the numbers of all the documents that contain that term, and* the frequency of the term in that document, unless frequencies are omitted ({@link* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For+ * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Proximity data}. For* each term in the dictionary, the positions that the term occurs in each document. Note that* this will not exist if all fields in all documents omit position data.* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For@@ -255,27 +255,27 @@* <td>The stored fields for documents</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Dictionary}</td>* <td>.tim</td>* <td>The term dictionary, stores term info</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Index}</td>* <td>.tip</td>* <td>The index into the Term Dictionary</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Frequencies}</td>* <td>.doc</td>* <td>Contains the list of docs which contain each term along with frequency</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Positions}</td>* <td>.pos</td>* <td>Stores position information about where a term occurs in the index</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>+ * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Payloads}</td>* <td>.pay</td>* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>* </tr>diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.javaindex 853e8f9b0c7..ea9df193413 100644--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java@@ -21,8 +21,8 @@ import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.Objects;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;import org.apache.lucene.index.ImpactsEnum;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;@@ -401,10 +401,10 @@ public class PhraseQuery extends Query {/*** A guess of the average number of simple operations for the initial seek and buffer refill per* document for the positions of a term. See also {@link- * Lucene90PostingsReader.BlockImpactsPostingsEnum#nextPosition()}.+ * Lucene99PostingsReader.BlockImpactsPostingsEnum#nextPosition()}.** <p>Aside: Instead of being constant this could depend among others on {@link- * Lucene90PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link+ * Lucene99PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block* size of the device storing the index.@@ -413,7 +413,7 @@ public class PhraseQuery extends Query {/*** Number of simple operations in {@link- * Lucene90PostingsReader.BlockImpactsPostingsEnum#nextPosition()} when no seek or buffer refill+ * Lucene99PostingsReader.BlockImpactsPostingsEnum#nextPosition()} when no seek or buffer refill* is done.*/private static final int TERM_OPS_PER_POS = 7;diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex 15681e8a2e5..0fd87daff7d 100644--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -13,4 +13,4 @@# See the License for the specific language governing permissions and# limitations under the License.-org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat+org.apache.lucene.codecs.lucene99.Lucene99PostingsFormatdiff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.javanew file mode 100644index 00000000000..29448ff1e41--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java@@ -0,0 +1,89 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.packed.PackedInts;++public class TestForDeltaUtil extends LuceneTestCase {++ public void testEncodeDecode() throws IOException {+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];++ for (int i = 0; i < iterations; ++i) {+ final int bpv = TestUtil.nextInt(random(), 1, 31 - 7);+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ values[i * ForUtil.BLOCK_SIZE + j] =+ RandomNumbers.randomIntBetween(random(), 1, (int) PackedInts.maxValue(bpv));+ }+ }++ final Directory d = new ByteBuffersDirectory();+ final long endPointer;++ {+ // encode+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());++ for (int i = 0; i < iterations; ++i) {+ long[] source = new long[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];+ }+ forDeltaUtil.encodeDeltas(source, out);+ }+ endPointer = out.getFilePointer();+ out.close();+ }++ {+ // decode+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());+ for (int i = 0; i < iterations; ++i) {+ long base = 0;+ final long[] restored = new long[ForUtil.BLOCK_SIZE];+ forDeltaUtil.decodeAndPrefixSum(in, base, restored);+ final long[] expected = new long[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ expected[j] = values[i * ForUtil.BLOCK_SIZE + j];+ if (j > 0) {+ expected[j] += expected[j - 1];+ } else {+ expected[j] += base;+ }+ }+ assertArrayEquals(Arrays.toString(restored), expected, restored);+ }+ assertEquals(endPointer, in.getFilePointer());+ in.close();+ }++ d.close();+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.javasimilarity index 98%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestForUtil.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.javaindex 363226423ee..2f179c743af 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestForUtil.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.codecs.lucene99;import com.carrotsearch.randomizedtesting.generators.RandomNumbers;import java.io.IOException;diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.javasimilarity index 91%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PostingsFormat.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.javaindex 90a9ebe5369..99c0e0a6ae2 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90PostingsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java@@ -14,7 +14,9 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.codecs.lucene99;++import static org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.readImpacts;import java.io.IOException;import java.util.Arrays;@@ -22,9 +24,9 @@ import java.util.Collections;import java.util.List;import org.apache.lucene.codecs.Codec;import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.lucene90.Lucene90ScoreSkipReader.MutableImpactList;import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;import org.apache.lucene.codecs.lucene90.blocktree.Stats;+import org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.index.DirectoryReader;@@ -40,8 +42,8 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.index.BasePostingsFormatTestCase;import org.apache.lucene.tests.util.TestUtil;-public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene90PostingsFormat());+public class TestLucene99PostingsFormat extends BasePostingsFormatTestCase {+ private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene99PostingsFormat());@Overrideprotected Codec getCodec() {@@ -77,7 +79,7 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {expectThrows(IllegalArgumentException.class,() -> {- new Lucene90PostingsFormat(minItemsInBlock, maxItemsInBlock);+ new Lucene99PostingsFormat(minItemsInBlock, maxItemsInBlock);});}@@ -131,13 +133,12 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {}try (Directory dir = newDirectory()) {try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {- Lucene90SkipWriter.writeImpacts(acc, out);+ Lucene99SkipWriter.writeImpacts(acc, out);}try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {byte[] b = new byte[Math.toIntExact(in.length())];in.readBytes(b, 0, b.length);- List<Impact> impacts2 =- Lucene90ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new MutableImpactList());+ List<Impact> impacts2 = readImpacts(new ByteArrayDataInput(b), new MutableImpactList());assertEquals(impacts, impacts2);}}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.javasimilarity index 73%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestPForUtil.javarename to lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.javaindex a7b37030ad1..16f719e93b2 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestPForUtil.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene90;+package org.apache.lucene.codecs.lucene99;import com.carrotsearch.randomizedtesting.generators.RandomNumbers;import java.io.IOException;@@ -62,42 +62,6 @@ public class TestPForUtil extends LuceneTestCase {d.close();}- public void testDeltaEncodeDecode() throws IOException {- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);- // cap at 31 - 7 bpv to ensure we don't overflow when working with deltas (i.e., 128 24 bit- // values treated as deltas will result in a final value that can fit in 31 bits)- final int[] values = createTestData(iterations, 31 - 7);-- final Directory d = new ByteBuffersDirectory();- final long endPointer = encodeTestData(iterations, values, d);-- IndexInput in = d.openInput("test.bin", IOContext.READONCE);- final PForUtil pForUtil = new PForUtil(new ForUtil());- for (int i = 0; i < iterations; ++i) {- if (random().nextInt(5) == 0) {- pForUtil.skip(in);- continue;- }- long base = 0;- final long[] restored = new long[ForUtil.BLOCK_SIZE];- pForUtil.decodeAndPrefixSum(in, base, restored);- final long[] expected = new long[ForUtil.BLOCK_SIZE];- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- expected[j] = values[i * ForUtil.BLOCK_SIZE + j];- if (j > 0) {- expected[j] += expected[j - 1];- } else {- expected[j] += base;- }- }- assertArrayEquals(Arrays.toString(restored), expected, restored);- }- assertEquals(endPointer, in.getFilePointer());- in.close();-- d.close();- }-private int[] createTestData(int iterations, int maxBpv) {final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.javaindex 414d360d5d1..2a5d8fffd01 100644--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java@@ -21,8 +21,8 @@ import java.io.IOException;import java.util.Collection;import java.util.Collections;import java.util.Objects;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.PostingsEnum;@@ -261,10 +261,10 @@ class TermIntervalsSource extends IntervalsSource {/*** A guess of the average number of simple operations for the initial seek and buffer refill per* document for the positions of a term. See also {@link- * Lucene90PostingsReader.EverythingEnum#nextPosition()}.+ * Lucene99PostingsReader.EverythingEnum#nextPosition()}.** <p>Aside: Instead of being constant this could depend among others on {@link- * Lucene90PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link+ * Lucene99PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block* size of the device storing the index.@@ -272,7 +272,7 @@ class TermIntervalsSource extends IntervalsSource {private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;/**- * Number of simple operations in {@link Lucene90PostingsReader.EverythingEnum#nextPosition()}+ * Number of simple operations in {@link Lucene99PostingsReader.EverythingEnum#nextPosition()}* when no seek or buffer refill is done.*/private static final int TERM_OPS_PER_POS = 7;diff --git a/lucene/suggest/src/java/module-info.java b/lucene/suggest/src/java/module-info.javaindex 4444a9dd2f1..673fa391213 100644--- a/lucene/suggest/src/java/module-info.java+++ b/lucene/suggest/src/java/module-info.java@@ -30,7 +30,8 @@ module org.apache.lucene.suggest {provides org.apache.lucene.codecs.PostingsFormat withorg.apache.lucene.search.suggest.document.Completion50PostingsFormat,org.apache.lucene.search.suggest.document.Completion84PostingsFormat,- org.apache.lucene.search.suggest.document.Completion90PostingsFormat;+ org.apache.lucene.search.suggest.document.Completion90PostingsFormat,+ org.apache.lucene.search.suggest.document.Completion99PostingsFormat;provides org.apache.lucene.analysis.TokenFilterFactory withorg.apache.lucene.search.suggest.analyzing.SuggestStopFilterFactory;}diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.javaindex 02536cc7f26..e53e991db22 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java@@ -22,7 +22,7 @@ import org.apache.lucene.codecs.PostingsFormat;* {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code* org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat}. This format is only used for* backward-compatibility of the index format and cannot be used to write data, use {@link- * Completion90PostingsFormat} on new indices.+ * Completion99PostingsFormat} on new indices.** @lucene.experimental*/diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.javaindex 8551fc3a80a..76e29811834 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java@@ -22,7 +22,7 @@ import org.apache.lucene.codecs.PostingsFormat;* {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code* org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat}. This format is only used for* backward-compatibility of the index format and cannot be used to write data, use {@link- * Completion90PostingsFormat} on new indices.+ * Completion99PostingsFormat} on new indices.** @lucene.experimental*/diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.javaindex cf2a9103aa7..db45303bcd7 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java@@ -19,8 +19,10 @@ package org.apache.lucene.search.suggest.document;import org.apache.lucene.codecs.PostingsFormat;/**- * {@link CompletionPostingsFormat} for {@link- * org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat}+ * {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code+ * org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat}. This format is only used for+ * backward-compatibility of the index format and cannot be used to write data, use {@link+ * Completion90PostingsFormat} on new indices.** @lucene.experimental*/diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.javanew file mode 100644index 00000000000..4449d754e29--- /dev/null+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java@@ -0,0 +1,45 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search.suggest.document;++import org.apache.lucene.codecs.PostingsFormat;++/**+ * {@link CompletionPostingsFormat} for {@link+ * org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}+ *+ * @lucene.experimental+ */+public class Completion99PostingsFormat extends CompletionPostingsFormat {+ /** Creates a {@link Completion99PostingsFormat} that will load the completion FST on-heap. */+ public Completion99PostingsFormat() {+ this(FSTLoadMode.ON_HEAP);+ }++ /**+ * Creates a {@link Completion99PostingsFormat} that will use the provided <code>fstLoadMode+ * </code> to determine if the completion FST should be loaded on or off heap.+ */+ public Completion99PostingsFormat(FSTLoadMode fstLoadMode) {+ super("Completion99", fstLoadMode);+ }++ @Override+ protected PostingsFormat delegatePostingsFormat() {+ return PostingsFormat.forName("Lucene99");+ }+}diff --git a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex a4070471047..81d1ed9465d 100644--- a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -33,3 +33,4 @@org.apache.lucene.search.suggest.document.Completion50PostingsFormatorg.apache.lucene.search.suggest.document.Completion84PostingsFormatorg.apache.lucene.search.suggest.document.Completion90PostingsFormat+org.apache.lucene.search.suggest.document.Completion99PostingsFormatdiff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.javaindex 682363f29cb..30201a8eac8 100644--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java@@ -965,7 +965,7 @@ public class TestSuggestField extends LuceneTestCase {new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {CompletionPostingsFormat.FSTLoadMode fstLoadMode =RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());- PostingsFormat postingsFormat = new Completion90PostingsFormat(fstLoadMode);+ PostingsFormat postingsFormat = new Completion99PostingsFormat(fstLoadMode);@Overridepublic PostingsFormat postingsFormat() {diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.javaindex 3c21de0f128..0a0c8f673db 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java@@ -28,9 +28,9 @@ import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; // javadocs-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -38,7 +38,7 @@ import org.apache.lucene.index.SegmentWriteState;// any PostingsFormat and make it ord-able.../**- * Customized version of {@link Lucene90PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.+ * Customized version of {@link Lucene99PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.*/public final class LuceneFixedGap extends PostingsFormat {final int termIndexInterval;@@ -54,7 +54,7 @@ public final class LuceneFixedGap extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase docs = new Lucene90PostingsWriter(state);+ PostingsWriterBase docs = new Lucene99PostingsWriter(state);// TODO: should we make the terms index more easily// pluggable? Ie so that this codec would record which@@ -91,7 +91,7 @@ public final class LuceneFixedGap extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postings = new Lucene90PostingsReader(state);+ PostingsReaderBase postings = new Lucene99PostingsReader(state);TermsIndexReaderBase indexReader;boolean success = false;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.javaindex 6a38882c153..a864263b7c3 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; // javadocs-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;// any PostingsFormat and make it ord-able.../**- * Customized version of {@link Lucene90PostingsFormat} that uses {@link+ * Customized version of {@link Lucene99PostingsFormat} that uses {@link* VariableGapTermsIndexWriter} with a fixed interval, but forcing high docfreq terms to be indexed* terms.*/@@ -59,7 +59,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase docs = new Lucene90PostingsWriter(state);+ PostingsWriterBase docs = new Lucene99PostingsWriter(state);// TODO: should we make the terms index more easily// pluggable? Ie so that this codec would record which@@ -100,7 +100,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postings = new Lucene90PostingsReader(state);+ PostingsReaderBase postings = new Lucene99PostingsReader(state);TermsIndexReaderBase indexReader;boolean success = false;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.javaindex 838d2968c7f..667acf0ceb3 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat; // javadocs-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;// any PostingsFormat and make it ord-able.../**- * Customized version of {@link Lucene90PostingsFormat} that uses {@link+ * Customized version of {@link Lucene99PostingsFormat} that uses {@link* VariableGapTermsIndexWriter} with a fixed interval.*/public final class LuceneVarGapFixedInterval extends PostingsFormat {@@ -56,7 +56,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase docs = new Lucene90PostingsWriter(state);+ PostingsWriterBase docs = new Lucene99PostingsWriter(state);// TODO: should we make the terms index more easily// pluggable? Ie so that this codec would record which@@ -95,7 +95,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postings = new Lucene90PostingsReader(state);+ PostingsReaderBase postings = new Lucene99PostingsReader(state);TermsIndexReaderBase indexReader;boolean success = false;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.javaindex 9243ce95059..10e5ec96332 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java@@ -35,10 +35,10 @@ import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.codecs.memory.FSTTermsReader;import org.apache.lucene.codecs.memory.FSTTermsWriter;import org.apache.lucene.index.FieldInfo;@@ -121,7 +121,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {random.nextInt(); // consume a random for buffersize- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(state);+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);final FieldsConsumer fields;final int t1 = random.nextInt(4);@@ -289,7 +289,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);}- PostingsReaderBase postingsReader = new Lucene90PostingsReader(state);+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);final FieldsProducer fields;final int t1 = random.nextInt(4);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.javaindex 7291c7e54da..0c469b392a2 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.PostingsReaderBase;import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsReader;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;import org.apache.lucene.codecs.uniformsplit.BlockDecoder;import org.apache.lucene.codecs.uniformsplit.BlockEncoder;import org.apache.lucene.codecs.uniformsplit.IndexDictionary;@@ -67,7 +67,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {@Overridepublic FieldsConsumer fieldsConsumer(SegmentWriteState segmentWriteState) throws IOException {- PostingsWriterBase postingsWriter = new Lucene90PostingsWriter(segmentWriteState);+ PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(segmentWriteState);boolean success = false;try {FieldsConsumer fieldsConsumer = createFieldsConsumer(segmentWriteState, postingsWriter);@@ -145,7 +145,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState segmentReadState) throws IOException {- PostingsReaderBase postingsReader = new Lucene90PostingsReader(segmentReadState);+ PostingsReaderBase postingsReader = new Lucene99PostingsReader(segmentReadState);boolean success = false;try {FieldsProducer fieldsProducer = createFieldsProducer(segmentReadState, postingsReader);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.javaindex b1c5e8d9b93..6d1d8573e5e 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java@@ -54,9 +54,9 @@ import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;-import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;import org.apache.lucene.codecs.lucene99.Lucene99Codec;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;import org.apache.lucene.document.BinaryDocValuesField;@@ -1244,7 +1244,7 @@ public final class TestUtil {* Lucene.*/public static PostingsFormat getDefaultPostingsFormat() {- return new Lucene90PostingsFormat();+ return new Lucene99PostingsFormat();}/**@@ -1255,7 +1255,7 @@ public final class TestUtil {*/public static PostingsFormat getDefaultPostingsFormat(int minItemsPerBlock, int maxItemsPerBlock) {- return new Lucene90PostingsFormat(minItemsPerBlock, maxItemsPerBlock);+ return new Lucene99PostingsFormat(minItemsPerBlock, maxItemsPerBlock);}/** Returns a random postings format that supports term ordinals */